2024-11-08 01:46:53,139 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 01:46:53,157 main DEBUG Took 0.016058 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 01:46:53,157 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 01:46:53,158 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 01:46:53,159 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 01:46:53,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,166 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 01:46:53,185 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,188 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,190 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,192 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,193 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,196 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,196 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,197 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,198 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,199 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,200 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,201 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,202 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,202 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,203 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,204 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,205 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,205 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 01:46:53,206 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,207 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 01:46:53,209 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 01:46:53,211 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 01:46:53,213 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 01:46:53,214 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 01:46:53,216 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 01:46:53,217 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 01:46:53,227 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 01:46:53,231 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 01:46:53,234 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 01:46:53,234 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 01:46:53,235 main DEBUG createAppenders(={Console}) 2024-11-08 01:46:53,236 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-08 01:46:53,236 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 01:46:53,236 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-08 01:46:53,237 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 01:46:53,238 main DEBUG OutputStream closed 2024-11-08 01:46:53,238 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 01:46:53,238 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 01:46:53,239 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-08 01:46:53,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 01:46:53,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 01:46:53,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 01:46:53,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 01:46:53,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 01:46:53,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 01:46:53,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 01:46:53,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 01:46:53,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 01:46:53,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 01:46:53,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 01:46:53,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 01:46:53,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 01:46:53,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 01:46:53,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 01:46:53,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 01:46:53,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 01:46:53,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 01:46:53,353 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 01:46:53,354 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-08 01:46:53,354 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 01:46:53,355 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-08T01:46:53,370 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-08 01:46:53,373 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 01:46:53,374 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T01:46:53,675 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c 2024-11-08T01:46:53,701 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=7103 2024-11-08T01:46:53,787 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T01:46:53,881 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:53,896 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/cluster_5086bd0b-5566-80e1-12ab-560dc62def85, deleteOnExit=true 2024-11-08T01:46:54,030 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/cluster_5086bd0b-5566-80e1-12ab-560dc62def85/zookeeper_0, clientPort=50081, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/cluster_5086bd0b-5566-80e1-12ab-560dc62def85/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/cluster_5086bd0b-5566-80e1-12ab-560dc62def85/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T01:46:54,044 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50081 2024-11-08T01:46:54,044 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:54,145 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 with version=8 2024-11-08T01:46:54,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/hbase-staging 2024-11-08T01:46:54,258 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T01:46:54,531 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8118b061437f:0 server-side Connection retries=45 2024-11-08T01:46:54,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:54,546 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:54,556 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T01:46:54,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:54,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T01:46:54,709 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T01:46:54,805 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T01:46:54,859 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T01:46:54,872 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T01:46:54,877 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T01:46:54,926 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14665 (auto-detected) 2024-11-08T01:46:54,927 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T01:46:54,960 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43675 2024-11-08T01:46:54,993 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43675 connecting to ZooKeeper ensemble=127.0.0.1:50081 2024-11-08T01:46:55,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436750x0, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T01:46:55,109 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43675-0x101181fee630000 connected 2024-11-08T01:46:55,210 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:55,211 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:55,222 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:46:55,227 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7, hbase.cluster.distributed=false 2024-11-08T01:46:55,259 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T01:46:55,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43675 2024-11-08T01:46:55,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43675 2024-11-08T01:46:55,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43675 2024-11-08T01:46:55,273 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43675 2024-11-08T01:46:55,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43675 2024-11-08T01:46:55,385 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8118b061437f:0 server-side Connection retries=45 2024-11-08T01:46:55,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:55,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:55,389 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T01:46:55,389 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:55,389 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T01:46:55,393 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T01:46:55,397 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T01:46:55,398 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42871 2024-11-08T01:46:55,401 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42871 connecting to ZooKeeper ensemble=127.0.0.1:50081 2024-11-08T01:46:55,402 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:55,404 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:55,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428710x0, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T01:46:55,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428710x0, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:46:55,424 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42871-0x101181fee630001 connected 2024-11-08T01:46:55,431 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T01:46:55,445 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.cache 2024-11-08T01:46:55,453 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T01:46:55,455 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T01:46:55,457 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T01:46:55,460 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T01:46:55,460 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T01:46:55,470 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T01:46:55,472 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T01:46:55,472 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.persistence is complete. 2024-11-08T01:46:55,478 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T01:46:55,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T01:46:55,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T01:46:55,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42871 2024-11-08T01:46:55,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42871 2024-11-08T01:46:55,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42871 2024-11-08T01:46:55,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42871 2024-11-08T01:46:55,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42871 2024-11-08T01:46:55,532 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8118b061437f:0 server-side Connection retries=45 2024-11-08T01:46:55,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:55,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:55,534 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T01:46:55,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:46:55,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T01:46:55,534 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T01:46:55,535 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T01:46:55,538 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43185 2024-11-08T01:46:55,541 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43185 connecting to ZooKeeper ensemble=127.0.0.1:50081 2024-11-08T01:46:55,542 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:55,543 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:55,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431850x0, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T01:46:55,564 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:431850x0, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:46:55,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43185-0x101181fee630002 connected 2024-11-08T01:46:55,565 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T01:46:55,569 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.cache 2024-11-08T01:46:55,569 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T01:46:55,569 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T01:46:55,570 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T01:46:55,573 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T01:46:55,573 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T01:46:55,574 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T01:46:55,575 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.persistence is complete. 2024-11-08T01:46:55,575 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T01:46:55,578 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T01:46:55,581 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T01:46:55,584 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T01:46:55,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43185 2024-11-08T01:46:55,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43185 2024-11-08T01:46:55,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43185 2024-11-08T01:46:55,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43185 2024-11-08T01:46:55,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43185 2024-11-08T01:46:55,620 DEBUG [M:0;8118b061437f:43675 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8118b061437f:43675 2024-11-08T01:46:55,622 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8118b061437f,43675,1731030414339 2024-11-08T01:46:55,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:46:55,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:46:55,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:46:55,641 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8118b061437f,43675,1731030414339 2024-11-08T01:46:55,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:55,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T01:46:55,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T01:46:55,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:55,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:55,689 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T01:46:55,691 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8118b061437f,43675,1731030414339 from backup master directory 2024-11-08T01:46:55,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:46:55,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8118b061437f,43675,1731030414339 2024-11-08T01:46:55,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:46:55,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:46:55,708 WARN [master/8118b061437f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T01:46:55,708 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8118b061437f,43675,1731030414339 2024-11-08T01:46:55,710 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T01:46:55,712 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T01:46:55,724 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/hbase.id] with ID: 8ff5d7bf-b0e5-468a-ab2f-2ea7274d2942 2024-11-08T01:46:55,724 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.tmp/hbase.id 2024-11-08T01:46:55,731 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/hbase.id] 2024-11-08T01:46:55,783 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:46:55,791 INFO [master/8118b061437f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T01:46:55,818 INFO [master/8118b061437f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-08T01:46:55,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:55,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:55,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:55,864 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T01:46:55,867 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T01:46:55,914 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T01:46:55,919 INFO [master/8118b061437f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:46:55,977 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store 2024-11-08T01:46:56,008 INFO [master/8118b061437f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T01:46:56,012 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:46:56,014 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T01:46:56,015 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:46:56,015 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:46:56,017 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T01:46:56,018 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:46:56,018 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:46:56,019 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731030416014Disabling compacts and flushes for region at 1731030416014Disabling writes for close at 1731030416017 (+3 ms)Writing region close event to WAL at 1731030416018 (+1 ms)Closed at 1731030416018 2024-11-08T01:46:56,020 WARN [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/.initializing 2024-11-08T01:46:56,020 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/WALs/8118b061437f,43675,1731030414339 2024-11-08T01:46:56,033 INFO [master/8118b061437f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T01:46:56,051 INFO [master/8118b061437f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8118b061437f%2C43675%2C1731030414339, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/WALs/8118b061437f,43675,1731030414339, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/oldWALs, maxLogs=10 2024-11-08T01:46:56,098 INFO [master/8118b061437f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/WALs/8118b061437f,43675,1731030414339/8118b061437f%2C43675%2C1731030414339.1731030416056 2024-11-08T01:46:56,104 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:46:56,105 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:46:56,106 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:46:56,111 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,113 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,168 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T01:46:56,217 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:56,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:56,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T01:46:56,233 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:56,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:46:56,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T01:46:56,243 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:56,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:46:56,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T01:46:56,253 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:56,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:46:56,254 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,259 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,260 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,265 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,265 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,269 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T01:46:56,272 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:46:56,293 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T01:46:56,294 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66126400, jitterRate=-0.014639854431152344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T01:46:56,305 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731030416130Initializing all the Stores at 1731030416131 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030416132 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030416134 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030416140 (+6 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030416140Cleaning up temporary data from old regions at 1731030416266 (+126 ms)Region opened successfully at 1731030416305 (+39 ms) 2024-11-08T01:46:56,307 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T01:46:56,364 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8118b061437f/172.17.0.2:0 2024-11-08T01:46:56,418 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T01:46:56,435 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T01:46:56,435 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T01:46:56,439 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T01:46:56,440 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T01:46:56,448 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T01:46:56,448 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T01:46:56,484 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T01:46:56,495 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T01:46:56,541 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T01:46:56,545 INFO [master/8118b061437f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T01:46:56,549 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T01:46:56,562 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T01:46:56,565 INFO [master/8118b061437f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T01:46:56,570 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T01:46:56,583 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T01:46:56,585 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T01:46:56,593 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T01:46:56,620 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T01:46:56,632 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T01:46:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T01:46:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T01:46:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T01:46:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,669 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8118b061437f,43675,1731030414339, sessionid=0x101181fee630000, setting cluster-up flag (Was=false) 2024-11-08T01:46:56,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,730 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T01:46:56,732 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8118b061437f,43675,1731030414339 2024-11-08T01:46:56,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:56,783 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T01:46:56,785 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8118b061437f,43675,1731030414339 2024-11-08T01:46:56,790 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T01:46:56,802 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(746): ClusterId : 8ff5d7bf-b0e5-468a-ab2f-2ea7274d2942 2024-11-08T01:46:56,805 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T01:46:56,802 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(746): ClusterId : 8ff5d7bf-b0e5-468a-ab2f-2ea7274d2942 2024-11-08T01:46:56,809 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T01:46:56,828 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T01:46:56,828 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T01:46:56,829 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T01:46:56,829 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T01:46:56,855 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T01:46:56,855 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T01:46:56,856 DEBUG [RS:0;8118b061437f:42871 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a82ade6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8118b061437f/172.17.0.2:0 2024-11-08T01:46:56,857 DEBUG [RS:1;8118b061437f:43185 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e1af88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8118b061437f/172.17.0.2:0 2024-11-08T01:46:56,875 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;8118b061437f:43185 2024-11-08T01:46:56,880 INFO [RS:1;8118b061437f:43185 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T01:46:56,880 INFO [RS:1;8118b061437f:43185 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T01:46:56,884 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8118b061437f:42871 2024-11-08T01:46:56,885 INFO [RS:0;8118b061437f:42871 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T01:46:56,885 INFO [RS:0;8118b061437f:42871 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T01:46:56,885 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T01:46:56,888 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(2659): reportForDuty to master=8118b061437f,43675,1731030414339 with port=42871, startcode=1731030415338 2024-11-08T01:46:56,894 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T01:46:56,880 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T01:46:56,900 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(2659): reportForDuty to master=8118b061437f,43675,1731030414339 with port=43185, startcode=1731030415532 2024-11-08T01:46:56,907 DEBUG [RS:1;8118b061437f:43185 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T01:46:56,907 DEBUG [RS:0;8118b061437f:42871 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T01:46:56,908 INFO [master/8118b061437f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T01:46:56,917 INFO [master/8118b061437f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T01:46:56,924 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8118b061437f,43675,1731030414339 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T01:46:56,933 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:46:56,933 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:46:56,933 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:46:56,934 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:46:56,934 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8118b061437f:0, corePoolSize=10, maxPoolSize=10 2024-11-08T01:46:56,934 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:56,936 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8118b061437f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T01:46:56,936 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:56,949 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731030446949 2024-11-08T01:46:56,952 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T01:46:56,953 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T01:46:56,958 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T01:46:56,959 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T01:46:56,959 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T01:46:56,960 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T01:46:56,961 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T01:46:56,962 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T01:46:56,967 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:56,967 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T01:46:56,970 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:56,990 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T01:46:56,992 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T01:46:56,993 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T01:46:57,004 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T01:46:57,005 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T01:46:57,015 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.large.0-1731030417006,5,FailOnTimeoutGroup] 2024-11-08T01:46:57,017 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.small.0-1731030417016,5,FailOnTimeoutGroup] 2024-11-08T01:46:57,017 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,018 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T01:46:57,018 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59743, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T01:46:57,020 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T01:46:57,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48785, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T01:46:57,025 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,026 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,026 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43675 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T01:46:57,027 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:46:57,035 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43675 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8118b061437f,42871,1731030415338 2024-11-08T01:46:57,038 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43675 {}] master.ServerManager(517): Registering regionserver=8118b061437f,42871,1731030415338 2024-11-08T01:46:57,048 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:46:57,048 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T01:46:57,048 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T01:46:57,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:46:57,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T01:46:57,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T01:46:57,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,059 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T01:46:57,060 WARN [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T01:46:57,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T01:46:57,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T01:46:57,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T01:46:57,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T01:46:57,066 DEBUG [RS:0;8118b061437f:42871 {}] zookeeper.ZKUtil(111): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8118b061437f,42871,1731030415338 2024-11-08T01:46:57,066 WARN [RS:0;8118b061437f:42871 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T01:46:57,066 INFO [RS:0;8118b061437f:42871 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:46:57,067 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,42871,1731030415338 2024-11-08T01:46:57,070 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8118b061437f,42871,1731030415338] 2024-11-08T01:46:57,078 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T01:46:57,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T01:46:57,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T01:46:57,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T01:46:57,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:46:57,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:46:57,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T01:46:57,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T01:46:57,090 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T01:46:57,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T01:46:57,102 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T01:46:57,102 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74791739, jitterRate=0.11448375880718231}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T01:46:57,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731030417051Initializing all the Stores at 1731030417051Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030417051Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030417053 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030417053Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030417053Cleaning up temporary data from old regions at 1731030417089 (+36 ms)Region opened successfully at 1731030417108 (+19 ms) 2024-11-08T01:46:57,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T01:46:57,109 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T01:46:57,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T01:46:57,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T01:46:57,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T01:46:57,112 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T01:46:57,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731030417108Disabling compacts and flushes for region at 1731030417108Disabling writes for close at 1731030417109 (+1 ms)Writing region close event to WAL at 1731030417112 (+3 ms)Closed at 1731030417112 2024-11-08T01:46:57,117 INFO [RS:0;8118b061437f:42871 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T01:46:57,118 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T01:46:57,118 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T01:46:57,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T01:46:57,139 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T01:46:57,142 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T01:46:57,145 INFO [RS:0;8118b061437f:42871 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T01:46:57,152 INFO [RS:0;8118b061437f:42871 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T01:46:57,152 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,154 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T01:46:57,159 INFO [RS:0;8118b061437f:42871 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T01:46:57,161 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(2659): reportForDuty to master=8118b061437f,43675,1731030414339 with port=43185, startcode=1731030415532 2024-11-08T01:46:57,162 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,162 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,162 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,162 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,162 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,162 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43675 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8118b061437f,43185,1731030415532 2024-11-08T01:46:57,163 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,163 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43675 {}] master.ServerManager(517): Registering regionserver=8118b061437f,43185,1731030415532 2024-11-08T01:46:57,163 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8118b061437f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T01:46:57,163 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,163 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,163 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,164 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,164 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,164 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,164 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:46:57,164 DEBUG [RS:0;8118b061437f:42871 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:46:57,165 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:46:57,165 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T01:46:57,165 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T01:46:57,168 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,168 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,169 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,169 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,169 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,169 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,42871,1731030415338-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T01:46:57,195 INFO [RS:0;8118b061437f:42871 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T01:46:57,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T01:46:57,198 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8118b061437f,43185,1731030415532] 2024-11-08T01:46:57,198 DEBUG [RS:1;8118b061437f:43185 {}] zookeeper.ZKUtil(111): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8118b061437f,43185,1731030415532 2024-11-08T01:46:57,198 WARN [RS:1;8118b061437f:43185 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T01:46:57,198 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,42871,1731030415338-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,199 INFO [RS:1;8118b061437f:43185 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:46:57,199 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43185,1731030415532 2024-11-08T01:46:57,199 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,199 INFO [RS:0;8118b061437f:42871 {}] regionserver.Replication(171): 8118b061437f,42871,1731030415338 started 2024-11-08T01:46:57,210 INFO [RS:1;8118b061437f:43185 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T01:46:57,220 INFO [RS:1;8118b061437f:43185 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T01:46:57,225 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,226 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1482): Serving as 8118b061437f,42871,1731030415338, RpcServer on 8118b061437f/172.17.0.2:42871, sessionid=0x101181fee630001 2024-11-08T01:46:57,227 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T01:46:57,228 DEBUG [RS:0;8118b061437f:42871 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8118b061437f,42871,1731030415338 2024-11-08T01:46:57,228 DEBUG [RS:0;8118b061437f:42871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,42871,1731030415338' 2024-11-08T01:46:57,228 DEBUG [RS:0;8118b061437f:42871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T01:46:57,231 DEBUG [RS:0;8118b061437f:42871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T01:46:57,231 INFO [RS:1;8118b061437f:43185 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T01:46:57,231 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,232 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T01:46:57,232 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T01:46:57,232 DEBUG [RS:0;8118b061437f:42871 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8118b061437f,42871,1731030415338 2024-11-08T01:46:57,232 DEBUG [RS:0;8118b061437f:42871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,42871,1731030415338' 2024-11-08T01:46:57,232 DEBUG [RS:0;8118b061437f:42871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T01:46:57,233 DEBUG [RS:0;8118b061437f:42871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T01:46:57,234 DEBUG [RS:0;8118b061437f:42871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T01:46:57,234 INFO [RS:0;8118b061437f:42871 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T01:46:57,234 INFO [RS:0;8118b061437f:42871 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T01:46:57,238 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T01:46:57,239 INFO [RS:1;8118b061437f:43185 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T01:46:57,240 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,240 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,240 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,240 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,240 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,241 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,241 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8118b061437f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T01:46:57,241 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,241 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,241 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,241 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,242 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,242 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:46:57,242 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:46:57,242 DEBUG [RS:1;8118b061437f:43185 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:46:57,248 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,249 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,249 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,249 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,249 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,249 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43185,1731030415532-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T01:46:57,272 INFO [RS:1;8118b061437f:43185 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T01:46:57,273 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43185,1731030415532-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,273 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,273 INFO [RS:1;8118b061437f:43185 {}] regionserver.Replication(171): 8118b061437f,43185,1731030415532 started 2024-11-08T01:46:57,293 WARN [8118b061437f:43675 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T01:46:57,298 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:57,298 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1482): Serving as 8118b061437f,43185,1731030415532, RpcServer on 8118b061437f/172.17.0.2:43185, sessionid=0x101181fee630002 2024-11-08T01:46:57,299 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T01:46:57,299 DEBUG [RS:1;8118b061437f:43185 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8118b061437f,43185,1731030415532 2024-11-08T01:46:57,299 DEBUG [RS:1;8118b061437f:43185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,43185,1731030415532' 2024-11-08T01:46:57,302 DEBUG [RS:1;8118b061437f:43185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T01:46:57,303 DEBUG [RS:1;8118b061437f:43185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T01:46:57,304 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T01:46:57,304 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T01:46:57,305 DEBUG [RS:1;8118b061437f:43185 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8118b061437f,43185,1731030415532 2024-11-08T01:46:57,305 DEBUG [RS:1;8118b061437f:43185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,43185,1731030415532' 2024-11-08T01:46:57,305 DEBUG [RS:1;8118b061437f:43185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T01:46:57,306 DEBUG [RS:1;8118b061437f:43185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T01:46:57,307 DEBUG [RS:1;8118b061437f:43185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T01:46:57,307 INFO [RS:1;8118b061437f:43185 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T01:46:57,307 INFO [RS:1;8118b061437f:43185 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T01:46:57,341 INFO [RS:0;8118b061437f:42871 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T01:46:57,343 INFO [RS:0;8118b061437f:42871 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C42871%2C1731030415338, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,42871,1731030415338, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:46:57,378 INFO [RS:0;8118b061437f:42871 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,42871,1731030415338/8118b061437f%2C42871%2C1731030415338.1731030417347 2024-11-08T01:46:57,378 DEBUG [RS:0;8118b061437f:42871 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:46:57,408 INFO [RS:1;8118b061437f:43185 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T01:46:57,410 INFO [RS:1;8118b061437f:43185 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C43185%2C1731030415532, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43185,1731030415532, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:46:57,441 INFO [RS:1;8118b061437f:43185 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43185,1731030415532/8118b061437f%2C43185%2C1731030415532.1731030417413 2024-11-08T01:46:57,442 DEBUG [RS:1;8118b061437f:43185 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:46:57,547 DEBUG [8118b061437f:43675 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T01:46:57,561 DEBUG [8118b061437f:43675 {}] balancer.BalancerClusterState(204): Hosts are {8118b061437f=0} racks are {/default-rack=0} 2024-11-08T01:46:57,566 DEBUG [8118b061437f:43675 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T01:46:57,566 DEBUG [8118b061437f:43675 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T01:46:57,566 DEBUG [8118b061437f:43675 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T01:46:57,566 DEBUG [8118b061437f:43675 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T01:46:57,567 INFO [8118b061437f:43675 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T01:46:57,567 INFO [8118b061437f:43675 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T01:46:57,567 DEBUG [8118b061437f:43675 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T01:46:57,576 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8118b061437f,42871,1731030415338 2024-11-08T01:46:57,584 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8118b061437f,42871,1731030415338, state=OPENING 2024-11-08T01:46:57,646 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T01:46:57,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:57,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:57,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:46:57,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:46:57,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:46:57,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:46:57,659 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T01:46:57,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8118b061437f,42871,1731030415338}] 2024-11-08T01:46:57,833 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T01:46:57,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52633, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T01:46:57,847 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T01:46:57,848 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:46:57,849 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T01:46:57,850 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C42871%2C1731030415338.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,42871,1731030415338, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:46:57,878 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,42871,1731030415338/8118b061437f%2C42871%2C1731030415338.meta.1731030417853.meta 2024-11-08T01:46:57,879 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:46:57,879 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:46:57,881 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T01:46:57,884 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T01:46:57,891 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T01:46:57,895 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T01:46:57,895 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:46:57,896 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T01:46:57,896 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T01:46:57,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T01:46:57,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T01:46:57,899 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T01:46:57,900 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T01:46:57,900 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T01:46:57,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T01:46:57,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T01:46:57,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T01:46:57,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:57,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:46:57,905 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T01:46:57,906 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:46:57,907 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:46:57,908 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T01:46:57,908 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T01:46:57,908 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T01:46:57,909 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T01:46:57,910 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71120822, jitterRate=0.059782832860946655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T01:46:57,910 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T01:46:57,912 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731030417896Writing region info on filesystem at 1731030417897 (+1 ms)Initializing all the Stores at 1731030417897Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030417897Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030417897Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030417897Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030417897Cleaning up temporary data from old regions at 1731030417908 (+11 ms)Running coprocessor post-open hooks at 1731030417911 (+3 ms)Region opened successfully at 1731030417912 (+1 ms) 2024-11-08T01:46:57,918 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731030417825 2024-11-08T01:46:57,933 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T01:46:57,933 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T01:46:57,935 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8118b061437f,42871,1731030415338 2024-11-08T01:46:57,937 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8118b061437f,42871,1731030415338, state=OPEN 2024-11-08T01:46:58,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:46:58,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:46:58,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:46:58,190 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:46:58,190 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:46:58,191 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8118b061437f,42871,1731030415338 2024-11-08T01:46:58,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:46:58,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T01:46:58,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8118b061437f,42871,1731030415338 in 529 msec 2024-11-08T01:46:58,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T01:46:58,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0690 sec 2024-11-08T01:46:58,202 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T01:46:58,202 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T01:46:58,226 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T01:46:58,228 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8118b061437f,42871,1731030415338, seqNum=-1] 2024-11-08T01:46:58,254 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T01:46:58,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49257, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T01:46:58,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4570 sec 2024-11-08T01:46:58,286 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731030418286, completionTime=-1 2024-11-08T01:46:58,290 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T01:46:58,290 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T01:46:58,324 INFO [master/8118b061437f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T01:46:58,324 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731030478324 2024-11-08T01:46:58,324 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731030538324 2024-11-08T01:46:58,325 INFO [master/8118b061437f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-08T01:46:58,327 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T01:46:58,334 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43675,1731030414339-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:58,334 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43675,1731030414339-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:58,334 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43675,1731030414339-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:58,336 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8118b061437f:43675, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:58,336 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:58,337 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:58,344 DEBUG [master/8118b061437f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T01:46:58,364 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.656sec 2024-11-08T01:46:58,365 INFO [master/8118b061437f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T01:46:58,366 INFO [master/8118b061437f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T01:46:58,367 INFO [master/8118b061437f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T01:46:58,368 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T01:46:58,368 INFO [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T01:46:58,369 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43675,1731030414339-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T01:46:58,369 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43675,1731030414339-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T01:46:58,370 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T01:46:58,371 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T01:46:58,371 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43675,1731030414339-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:46:58,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T01:46:58,419 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T01:46:58,420 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T01:46:58,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8118b061437f,43675,-1 for getting cluster id 2024-11-08T01:46:58,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T01:46:58,437 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff5d7bf-b0e5-468a-ab2f-2ea7274d2942' 2024-11-08T01:46:58,440 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T01:46:58,440 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff5d7bf-b0e5-468a-ab2f-2ea7274d2942" 2024-11-08T01:46:58,443 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5164b545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T01:46:58,443 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8118b061437f,43675,-1] 2024-11-08T01:46:58,446 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T01:46:58,448 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:46:58,449 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T01:46:58,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T01:46:58,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T01:46:58,460 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8118b061437f,42871,1731030415338, seqNum=-1] 2024-11-08T01:46:58,461 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T01:46:58,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T01:46:58,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8118b061437f,43675,1731030414339 2024-11-08T01:46:58,495 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T01:46:58,499 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8118b061437f,43675,1731030414339 2024-11-08T01:46:58,514 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28483214 2024-11-08T01:46:58,516 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T01:46:58,520 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T01:46:58,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731030418491', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T01:46:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 2024-11-08T01:46:58,540 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T01:46:58,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731030418491" procId is: 4 2024-11-08T01:46:58,542 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:58,544 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T01:46:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:46:58,556 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76c7d6f380f940e565bef23e566d54f4, NAME => 'testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731030418491', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:46:58,563 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:46:58,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(1722): Closing 76c7d6f380f940e565bef23e566d54f4, disabling compactions & flushes 2024-11-08T01:46:58,564 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:46:58,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:46:58,565 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. after waiting 0 ms 2024-11-08T01:46:58,565 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:46:58,565 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:46:58,565 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731030418491-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76c7d6f380f940e565bef23e566d54f4: Waiting for close lock at 1731030418564Disabling compacts and flushes for region at 1731030418564Disabling writes for close at 1731030418565 (+1 ms)Writing region close event to WAL at 1731030418565Closed at 1731030418565 2024-11-08T01:46:58,567 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T01:46:58,572 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731030418567"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731030418567"}]},"ts":"1731030418567"} 2024-11-08T01:46:58,577 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T01:46:58,578 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T01:46:58,580 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731030418491","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731030418578"}]},"ts":"1731030418578"} 2024-11-08T01:46:58,584 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731030418491, state=ENABLING in hbase:meta 2024-11-08T01:46:58,585 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {8118b061437f=0} racks are {/default-rack=0} 2024-11-08T01:46:58,585 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T01:46:58,585 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T01:46:58,585 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T01:46:58,585 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T01:46:58,585 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T01:46:58,585 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T01:46:58,585 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T01:46:58,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, ASSIGN}] 2024-11-08T01:46:58,588 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, ASSIGN 2024-11-08T01:46:58,589 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, ASSIGN; state=OFFLINE, location=8118b061437f,42871,1731030415338; forceNewPlan=false, retain=false 2024-11-08T01:46:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:46:58,742 INFO [8118b061437f:43675 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T01:46:58,743 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=76c7d6f380f940e565bef23e566d54f4, regionState=OPENING, regionLocation=8118b061437f,42871,1731030415338 2024-11-08T01:46:58,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, ASSIGN because future has completed 2024-11-08T01:46:58,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,42871,1731030415338}] 2024-11-08T01:46:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:46:58,910 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:46:58,911 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 76c7d6f380f940e565bef23e566d54f4, NAME => 'testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:46:58,911 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731030418491 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,911 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:46:58,912 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,912 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,913 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,917 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c7d6f380f940e565bef23e566d54f4 columnFamilyName family 2024-11-08T01:46:58,917 DEBUG [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:46:58,918 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.HStore(327): Store=76c7d6f380f940e565bef23e566d54f4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:46:58,918 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,920 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,920 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,920 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,920 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,922 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,928 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T01:46:58,928 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 76c7d6f380f940e565bef23e566d54f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74141064, jitterRate=0.10478794574737549}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T01:46:58,929 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:46:58,930 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 76c7d6f380f940e565bef23e566d54f4: Running coprocessor pre-open hook at 1731030418912Writing region info on filesystem at 1731030418912Initializing all the Stores at 1731030418913 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030418913Cleaning up temporary data from old regions at 1731030418921 (+8 ms)Running coprocessor post-open hooks at 1731030418929 (+8 ms)Region opened successfully at 1731030418929 2024-11-08T01:46:58,931 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4., pid=6, masterSystemTime=1731030418907 2024-11-08T01:46:58,935 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:46:58,935 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:46:58,935 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=76c7d6f380f940e565bef23e566d54f4, regionState=OPEN, openSeqNum=2, regionLocation=8118b061437f,42871,1731030415338 2024-11-08T01:46:58,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,42871,1731030415338 because future has completed 2024-11-08T01:46:58,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T01:46:58,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,42871,1731030415338 in 187 msec 2024-11-08T01:46:58,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T01:46:58,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, ASSIGN in 355 msec 2024-11-08T01:46:58,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T01:46:58,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731030418491","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731030418946"}]},"ts":"1731030418946"} 2024-11-08T01:46:58,949 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731030418491, state=ENABLED in hbase:meta 2024-11-08T01:46:58,950 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T01:46:58,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731030418491 in 418 msec 2024-11-08T01:46:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:46:59,180 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731030418491 get assigned. Timeout = 60000ms 2024-11-08T01:46:59,180 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731030418491 completed 2024-11-08T01:46:59,184 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T01:46:59,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731030418491 assigned to meta. Checking AM states. 2024-11-08T01:46:59,189 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T01:46:59,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731030418491 assigned. 2024-11-08T01:46:59,197 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731030418491', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4., hostname=8118b061437f,42871,1731030415338, seqNum=2] 2024-11-08T01:46:59,207 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76c7d6f380f940e565bef23e566d54f4 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T01:46:59,229 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:46:59,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/.tmp/family/981dae7eb1db4c0e941e93d9eeef66d8 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T01:46:59,267 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/.tmp/family/981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:46:59,301 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/.tmp/family/981dae7eb1db4c0e941e93d9eeef66d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:46:59,303 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8, delay=952 ms 2024-11-08T01:46:59,305 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T01:46:59,311 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 76c7d6f380f940e565bef23e566d54f4 in 103ms, sequenceid=6, compaction requested=false 2024-11-08T01:46:59,313 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731030418491' 2024-11-08T01:46:59,315 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 76c7d6f380f940e565bef23e566d54f4: 2024-11-08T01:47:00,257 DEBUG [hfile-prefetch-1731030419303 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 981dae7eb1db4c0e941e93d9eeef66d8_0. 2024-11-08T01:47:00,261 DEBUG [hfile-prefetch-1731030419303 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 981dae7eb1db4c0e941e93d9eeef66d8_117. 2024-11-08T01:47:00,262 DEBUG [hfile-prefetch-1731030419303 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8, with total blocks 2, and data blocks 1 2024-11-08T01:47:00,262 DEBUG [hfile-prefetch-1731030419303 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T01:47:00,262 DEBUG [hfile-prefetch-1731030419303 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T01:47:00,263 DEBUG [hfile-prefetch-1731030419303 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8. Will sleep 100ms and try the verification again. 2024-11-08T01:47:00,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=76c7d6f380f940e565bef23e566d54f4, source=8118b061437f,42871,1731030415338, destination=8118b061437f,43185,1731030415532, warming up region on 8118b061437f,43185,1731030415532 2024-11-08T01:47:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T01:47:00,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=76c7d6f380f940e565bef23e566d54f4, source=8118b061437f,42871,1731030415338, destination=8118b061437f,43185,1731030415532, running balancer 2024-11-08T01:47:00,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56499, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T01:47:00,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, REOPEN/MOVE 2024-11-08T01:47:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(7855): Warmup {ENCODED => 76c7d6f380f940e565bef23e566d54f4, NAME => 'testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:47:00,335 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, REOPEN/MOVE 2024-11-08T01:47:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:00,338 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76c7d6f380f940e565bef23e566d54f4, regionState=CLOSING, regionLocation=8118b061437f,42871,1731030415338 2024-11-08T01:47:00,338 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,339 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c7d6f380f940e565bef23e566d54f4 columnFamilyName family 2024-11-08T01:47:00,339 DEBUG [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:00,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, REOPEN/MOVE because future has completed 2024-11-08T01:47:00,343 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-08T01:47:00,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,42871,1731030415338}] 2024-11-08T01:47:00,344 DEBUG [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:47:00,344 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.HStore(327): Store=76c7d6f380f940e565bef23e566d54f4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:47:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(1722): Closing 76c7d6f380f940e565bef23e566d54f4, disabling compactions & flushes 2024-11-08T01:47:00,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. after waiting 0 ms 2024-11-08T01:47:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,351 DEBUG [StoreFileCloser-76c7d6f380f940e565bef23e566d54f4-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:47:00,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43185 {}] regionserver.HRegion(1676): Region close journal for 76c7d6f380f940e565bef23e566d54f4: Waiting for close lock at 1731030420346Disabling compacts and flushes for region at 1731030420346Disabling writes for close at 1731030420347 (+1 ms)Writing region close event to WAL at 1731030420355 (+8 ms)Closed at 1731030420355 2024-11-08T01:47:00,355 WARN [hfile-prefetch-1731030419303 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-08T01:47:00,356 DEBUG [hfile-prefetch-1731030419303 {}] hfile.PrefetchExecutor(121): Prefetch completed for 981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:47:00,362 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T01:47:00,363 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T01:47:00,363 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:00,363 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:00,378 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T01:47:00,378 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T01:47:00,378 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:00,378 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:00,391 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T01:47:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43675 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-08T01:47:00,391 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731030420378 2024-11-08T01:47:00,391 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T01:47:00,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T01:47:00,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731030420363 2024-11-08T01:47:00,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T01:47:00,502 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,503 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-08T01:47:00,503 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 76c7d6f380f940e565bef23e566d54f4, disabling compactions & flushes 2024-11-08T01:47:00,504 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,504 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,504 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. after waiting 0 ms 2024-11-08T01:47:00,504 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,506 DEBUG [StoreFileCloser-76c7d6f380f940e565bef23e566d54f4-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 981dae7eb1db4c0e941e93d9eeef66d8, starting offset: 0, end offset: 9223372036854775807 2024-11-08T01:47:00,508 DEBUG [StoreFileCloser-76c7d6f380f940e565bef23e566d54f4-family-1 {}] bucket.BucketCache(857): removed key 981dae7eb1db4c0e941e93d9eeef66d8_0 from back map with offset lock 2094080 in the evict process 2024-11-08T01:47:00,515 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T01:47:00,517 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,517 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 76c7d6f380f940e565bef23e566d54f4: Waiting for close lock at 1731030420503Running coprocessor pre-close hooks at 1731030420503Disabling compacts and flushes for region at 1731030420503Disabling writes for close at 1731030420504 (+1 ms)Writing region close event to WAL at 1731030420509 (+5 ms)Running coprocessor post-close hooks at 1731030420516 (+7 ms)Closed at 1731030420517 (+1 ms) 2024-11-08T01:47:00,518 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 76c7d6f380f940e565bef23e566d54f4 move to 8118b061437f,43185,1731030415532 record at close sequenceid=6 2024-11-08T01:47:00,523 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,523 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76c7d6f380f940e565bef23e566d54f4, regionState=CLOSED 2024-11-08T01:47:00,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,42871,1731030415338 because future has completed 2024-11-08T01:47:00,537 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, REOPEN/MOVE; state=CLOSED, location=8118b061437f,43185,1731030415532; forceNewPlan=false, retain=false 2024-11-08T01:47:00,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T01:47:00,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,42871,1731030415338 in 189 msec 2024-11-08T01:47:00,592 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T01:47:00,592 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T01:47:00,592 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:00,592 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:00,595 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T01:47:00,595 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731030420592 2024-11-08T01:47:00,595 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T01:47:00,688 INFO [8118b061437f:43675 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T01:47:00,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76c7d6f380f940e565bef23e566d54f4, regionState=OPENING, regionLocation=8118b061437f,43185,1731030415532 2024-11-08T01:47:00,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, REOPEN/MOVE because future has completed 2024-11-08T01:47:00,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,43185,1731030415532}] 2024-11-08T01:47:00,850 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,850 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 76c7d6f380f940e565bef23e566d54f4, NAME => 'testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:47:00,851 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731030418491 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,851 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:00,851 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,851 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,853 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,854 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c7d6f380f940e565bef23e566d54f4 columnFamilyName family 2024-11-08T01:47:00,854 DEBUG [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:00,857 DEBUG [StoreFileOpener-76c7d6f380f940e565bef23e566d54f4-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8, delay=928 ms 2024-11-08T01:47:00,859 DEBUG [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:47:00,859 INFO [StoreOpener-76c7d6f380f940e565bef23e566d54f4-1 {}] regionserver.HStore(327): Store=76c7d6f380f940e565bef23e566d54f4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:47:00,859 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,861 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,861 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,861 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,861 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,868 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,868 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 76c7d6f380f940e565bef23e566d54f4; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65867882, jitterRate=-0.018492072820663452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T01:47:00,869 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,870 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 76c7d6f380f940e565bef23e566d54f4: Running coprocessor pre-open hook at 1731030420852Writing region info on filesystem at 1731030420852Initializing all the Stores at 1731030420852Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030420852Cleaning up temporary data from old regions at 1731030420862 (+10 ms)Running coprocessor post-open hooks at 1731030420869 (+7 ms)Region opened successfully at 1731030420870 (+1 ms) 2024-11-08T01:47:00,871 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4., pid=9, masterSystemTime=1731030420846 2024-11-08T01:47:00,875 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,875 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76c7d6f380f940e565bef23e566d54f4, regionState=OPEN, openSeqNum=10, regionLocation=8118b061437f,43185,1731030415532 2024-11-08T01:47:00,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,43185,1731030415532 because future has completed 2024-11-08T01:47:00,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-08T01:47:00,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 76c7d6f380f940e565bef23e566d54f4, server=8118b061437f,43185,1731030415532 in 188 msec 2024-11-08T01:47:00,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731030418491, region=76c7d6f380f940e565bef23e566d54f4, REOPEN/MOVE in 553 msec 2024-11-08T01:47:00,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T01:47:00,894 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T01:47:00,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:00,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:00,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:00,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T01:47:00,902 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T01:47:00,902 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-08T01:47:00,903 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8118b061437f,43675,1731030414339 2024-11-08T01:47:00,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:00,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:00,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:00,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:00,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:00,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:00,917 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T01:47:00,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:00,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:00,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:00,920 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T01:47:00,920 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:00,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:00,922 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8118b061437f,42871,1731030415338' ***** 2024-11-08T01:47:00,922 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T01:47:00,922 INFO [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T01:47:00,923 INFO [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.persistence 2024-11-08T01:47:00,923 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T01:47:00,923 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T01:47:00,923 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T01:47:00,923 INFO [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T01:47:00,923 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T01:47:00,924 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8118b061437f,43185,1731030415532' ***** 2024-11-08T01:47:00,924 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T01:47:00,924 INFO [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T01:47:00,924 INFO [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/bucket.persistence 2024-11-08T01:47:00,924 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T01:47:00,924 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T01:47:00,924 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T01:47:00,924 INFO [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T01:47:00,924 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T01:47:00,933 DEBUG [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1488): Thread RS:0;8118b061437f:42871 started persisting bucket cache to file 2024-11-08T01:47:00,934 DEBUG [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:00,934 DEBUG [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:00,935 DEBUG [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1488): Thread RS:1;8118b061437f:43185 started persisting bucket cache to file 2024-11-08T01:47:00,935 DEBUG [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:00,935 DEBUG [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:00,938 DEBUG [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T01:47:00,938 DEBUG [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731030420933 2024-11-08T01:47:00,938 DEBUG [RS:0;8118b061437f:42871 {}] bucket.BucketCache(1510): Thread RS:0;8118b061437f:42871 finished persisting bucket cache to file, renaming 2024-11-08T01:47:00,939 INFO [RS:0;8118b061437f:42871 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T01:47:00,939 INFO [RS:0;8118b061437f:42871 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T01:47:00,939 INFO [RS:0;8118b061437f:42871 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T01:47:00,939 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(959): stopping server 8118b061437f,42871,1731030415338 2024-11-08T01:47:00,940 INFO [RS:0;8118b061437f:42871 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T01:47:00,940 INFO [RS:0;8118b061437f:42871 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8118b061437f:42871. 2024-11-08T01:47:00,940 DEBUG [RS:0;8118b061437f:42871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:00,940 DEBUG [RS:0;8118b061437f:42871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:00,940 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T01:47:00,940 INFO [RS:0;8118b061437f:42871 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T01:47:00,940 INFO [RS:0;8118b061437f:42871 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T01:47:00,940 INFO [RS:0;8118b061437f:42871 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T01:47:00,941 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T01:47:00,941 DEBUG [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T01:47:00,941 DEBUG [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731030420935 2024-11-08T01:47:00,941 DEBUG [RS:1;8118b061437f:43185 {}] bucket.BucketCache(1510): Thread RS:1;8118b061437f:43185 finished persisting bucket cache to file, renaming 2024-11-08T01:47:00,942 INFO [RS:1;8118b061437f:43185 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T01:47:00,942 INFO [RS:1;8118b061437f:43185 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T01:47:00,942 INFO [RS:1;8118b061437f:43185 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T01:47:00,942 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(3091): Received CLOSE for 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,943 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(959): stopping server 8118b061437f,43185,1731030415532 2024-11-08T01:47:00,943 INFO [RS:1;8118b061437f:43185 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T01:47:00,943 INFO [RS:1;8118b061437f:43185 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;8118b061437f:43185. 2024-11-08T01:47:00,943 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T01:47:00,943 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T01:47:00,943 DEBUG [RS:1;8118b061437f:43185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:00,943 DEBUG [RS:1;8118b061437f:43185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:00,943 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T01:47:00,944 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T01:47:00,944 DEBUG [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T01:47:00,944 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1325): Online Regions={76c7d6f380f940e565bef23e566d54f4=testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4.} 2024-11-08T01:47:00,944 DEBUG [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1351): Waiting on 76c7d6f380f940e565bef23e566d54f4 2024-11-08T01:47:00,944 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T01:47:00,944 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 76c7d6f380f940e565bef23e566d54f4, disabling compactions & flushes 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T01:47:00,945 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,945 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. after waiting 0 ms 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,945 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T01:47:00,946 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-08T01:47:00,949 DEBUG [StoreFileCloser-76c7d6f380f940e565bef23e566d54f4-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/family/981dae7eb1db4c0e941e93d9eeef66d8 2024-11-08T01:47:00,955 INFO [regionserver/8118b061437f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:00,961 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnRegionMove1731030418491/76c7d6f380f940e565bef23e566d54f4/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T01:47:00,961 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,961 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 76c7d6f380f940e565bef23e566d54f4: Waiting for close lock at 1731030420944Running coprocessor pre-close hooks at 1731030420945 (+1 ms)Disabling compacts and flushes for region at 1731030420945Disabling writes for close at 1731030420945Writing region close event to WAL at 1731030420950 (+5 ms)Running coprocessor post-close hooks at 1731030420961 (+11 ms)Closed at 1731030420961 2024-11-08T01:47:00,962 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4. 2024-11-08T01:47:00,968 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 45a6601ba577486d845d0b6056fc99de 2024-11-08T01:47:00,975 INFO [regionserver/8118b061437f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:00,988 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/45a6601ba577486d845d0b6056fc99de is 205, key is testBlockEvictionOnRegionMove1731030418491,,1731030418521.76c7d6f380f940e565bef23e566d54f4./info:regioninfo/1731030420875/Put/seqid=0 2024-11-08T01:47:00,989 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/45a6601ba577486d845d0b6056fc99de 2024-11-08T01:47:01,008 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8fa5adc17d0d414ba10ea88f37ca1abe 2024-11-08T01:47:01,030 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/ns/8fa5adc17d0d414ba10ea88f37ca1abe is 43, key is default/ns:d/1731030418262/Put/seqid=0 2024-11-08T01:47:01,031 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/ns/8fa5adc17d0d414ba10ea88f37ca1abe 2024-11-08T01:47:01,050 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d12c0cdfa5e642d88ac8a4914b076a23 2024-11-08T01:47:01,070 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/table/d12c0cdfa5e642d88ac8a4914b076a23 is 78, key is testBlockEvictionOnRegionMove1731030418491/table:state/1731030418946/Put/seqid=0 2024-11-08T01:47:01,071 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/table/d12c0cdfa5e642d88ac8a4914b076a23 2024-11-08T01:47:01,076 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/45a6601ba577486d845d0b6056fc99de as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/45a6601ba577486d845d0b6056fc99de 2024-11-08T01:47:01,078 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/45a6601ba577486d845d0b6056fc99de, delay=960 ms 2024-11-08T01:47:01,083 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/45a6601ba577486d845d0b6056fc99de, entries=15, sequenceid=15, filesize=7.6 K 2024-11-08T01:47:01,084 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/ns/8fa5adc17d0d414ba10ea88f37ca1abe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/8fa5adc17d0d414ba10ea88f37ca1abe 2024-11-08T01:47:01,086 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/8fa5adc17d0d414ba10ea88f37ca1abe, delay=959 ms 2024-11-08T01:47:01,088 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/8fa5adc17d0d414ba10ea88f37ca1abe, entries=2, sequenceid=15, filesize=5.0 K 2024-11-08T01:47:01,089 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/table/d12c0cdfa5e642d88ac8a4914b076a23 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/d12c0cdfa5e642d88ac8a4914b076a23 2024-11-08T01:47:01,091 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/d12c0cdfa5e642d88ac8a4914b076a23, delay=977 ms 2024-11-08T01:47:01,093 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/d12c0cdfa5e642d88ac8a4914b076a23, entries=2, sequenceid=15, filesize=5.3 K 2024-11-08T01:47:01,094 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 148ms, sequenceid=15, compaction requested=false 2024-11-08T01:47:01,094 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T01:47:01,095 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/45a6601ba577486d845d0b6056fc99de 2024-11-08T01:47:01,096 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/8fa5adc17d0d414ba10ea88f37ca1abe 2024-11-08T01:47:01,096 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/d12c0cdfa5e642d88ac8a4914b076a23 2024-11-08T01:47:01,101 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-08T01:47:01,102 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T01:47:01,102 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T01:47:01,102 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731030420944Running coprocessor pre-close hooks at 1731030420945 (+1 ms)Disabling compacts and flushes for region at 1731030420945Disabling writes for close at 1731030420945Obtaining lock to block concurrent updates at 1731030420946 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731030420946Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731030420946Flushing stores of hbase:meta,,1.1588230740 at 1731030420947 (+1 ms)Flushing 1588230740/info: creating writer at 1731030420947Flushing 1588230740/info: appending metadata at 1731030420986 (+39 ms)Flushing 1588230740/info: closing flushed file at 1731030420986Flushing 1588230740/ns: creating writer at 1731030420996 (+10 ms)Flushing 1588230740/ns: appending metadata at 1731030421028 (+32 ms)Flushing 1588230740/ns: closing flushed file at 1731030421028Flushing 1588230740/table: creating writer at 1731030421036 (+8 ms)Flushing 1588230740/table: appending metadata at 1731030421069 (+33 ms)Flushing 1588230740/table: closing flushed file at 1731030421069Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bc3f387: reopening flushed file at 1731030421075 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48ad59c1: reopening flushed file at 1731030421084 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3358f3e: reopening flushed file at 1731030421088 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 148ms, sequenceid=15, compaction requested=false at 1731030421094 (+6 ms)Writing region close event to WAL at 1731030421097 (+3 ms)Running coprocessor post-close hooks at 1731030421102 (+5 ms)Closed at 1731030421102 2024-11-08T01:47:01,103 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T01:47:01,144 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(976): stopping server 8118b061437f,43185,1731030415532; all regions closed. 2024-11-08T01:47:01,144 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(976): stopping server 8118b061437f,42871,1731030415338; all regions closed. 2024-11-08T01:47:01,149 DEBUG [RS:1;8118b061437f:43185 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:01,150 INFO [RS:1;8118b061437f:43185 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C43185%2C1731030415532:(num 1731030417413) 2024-11-08T01:47:01,150 DEBUG [RS:1;8118b061437f:43185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:01,150 INFO [RS:1;8118b061437f:43185 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:01,150 INFO [RS:1;8118b061437f:43185 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T01:47:01,150 INFO [RS:1;8118b061437f:43185 {}] hbase.ChoreService(370): Chore service for: regionserver/8118b061437f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T01:47:01,151 INFO [RS:1;8118b061437f:43185 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T01:47:01,151 INFO [regionserver/8118b061437f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T01:47:01,151 INFO [RS:1;8118b061437f:43185 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T01:47:01,151 INFO [RS:1;8118b061437f:43185 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T01:47:01,151 INFO [RS:1;8118b061437f:43185 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T01:47:01,151 DEBUG [RS:0;8118b061437f:42871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:01,151 INFO [RS:0;8118b061437f:42871 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C42871%2C1731030415338.meta:.meta(num 1731030417853) 2024-11-08T01:47:01,151 INFO [RS:1;8118b061437f:43185 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43185 2024-11-08T01:47:01,154 DEBUG [RS:0;8118b061437f:42871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:01,154 INFO [RS:0;8118b061437f:42871 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C42871%2C1731030415338:(num 1731030417347) 2024-11-08T01:47:01,154 DEBUG [RS:0;8118b061437f:42871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:01,154 INFO [RS:0;8118b061437f:42871 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:01,154 INFO [RS:0;8118b061437f:42871 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T01:47:01,154 INFO [RS:0;8118b061437f:42871 {}] hbase.ChoreService(370): Chore service for: regionserver/8118b061437f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T01:47:01,155 INFO [RS:0;8118b061437f:42871 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T01:47:01,155 INFO [RS:0;8118b061437f:42871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42871 2024-11-08T01:47:01,155 INFO [regionserver/8118b061437f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T01:47:01,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8118b061437f,43185,1731030415532 2024-11-08T01:47:01,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T01:47:01,169 INFO [RS:1;8118b061437f:43185 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T01:47:01,179 INFO [RS:0;8118b061437f:42871 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T01:47:01,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8118b061437f,42871,1731030415338 2024-11-08T01:47:01,180 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8118b061437f,42871,1731030415338] 2024-11-08T01:47:01,179 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f89d44772e0@10ffd2f7 rejected from java.util.concurrent.ThreadPoolExecutor@55fd4e8c[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-08T01:47:01,200 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8118b061437f,42871,1731030415338 already deleted, retry=false 2024-11-08T01:47:01,200 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8118b061437f,42871,1731030415338 expired; onlineServers=1 2024-11-08T01:47:01,201 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8118b061437f,43185,1731030415532] 2024-11-08T01:47:01,287 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8118b061437f,43185,1731030415532 already deleted, retry=false 2024-11-08T01:47:01,288 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8118b061437f,43185,1731030415532 expired; onlineServers=0 2024-11-08T01:47:01,288 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8118b061437f,43675,1731030414339' ***** 2024-11-08T01:47:01,288 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T01:47:01,288 INFO [M:0;8118b061437f:43675 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T01:47:01,288 INFO [M:0;8118b061437f:43675 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T01:47:01,288 DEBUG [M:0;8118b061437f:43675 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T01:47:01,289 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T01:47:01,289 DEBUG [M:0;8118b061437f:43675 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T01:47:01,289 DEBUG [master/8118b061437f:0:becomeActiveMaster-HFileCleaner.large.0-1731030417006 {}] cleaner.HFileCleaner(306): Exit Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.large.0-1731030417006,5,FailOnTimeoutGroup] 2024-11-08T01:47:01,289 DEBUG [master/8118b061437f:0:becomeActiveMaster-HFileCleaner.small.0-1731030417016 {}] cleaner.HFileCleaner(306): Exit Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.small.0-1731030417016,5,FailOnTimeoutGroup] 2024-11-08T01:47:01,289 INFO [M:0;8118b061437f:43675 {}] hbase.ChoreService(370): Chore service for: master/8118b061437f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T01:47:01,289 INFO [M:0;8118b061437f:43675 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T01:47:01,289 DEBUG [M:0;8118b061437f:43675 {}] master.HMaster(1795): Stopping service threads 2024-11-08T01:47:01,289 INFO [M:0;8118b061437f:43675 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T01:47:01,289 INFO [M:0;8118b061437f:43675 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T01:47:01,290 INFO [M:0;8118b061437f:43675 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T01:47:01,290 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T01:47:01,291 INFO [RS:1;8118b061437f:43185 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T01:47:01,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:01,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43185-0x101181fee630002, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:01,291 INFO [RS:1;8118b061437f:43185 {}] regionserver.HRegionServer(1031): Exiting; stopping=8118b061437f,43185,1731030415532; zookeeper connection closed. 2024-11-08T01:47:01,291 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@16715a2d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@16715a2d 2024-11-08T01:47:01,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T01:47:01,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:01,299 DEBUG [M:0;8118b061437f:43675 {}] zookeeper.ZKUtil(347): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T01:47:01,299 WARN [M:0;8118b061437f:43675 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T01:47:01,299 INFO [M:0;8118b061437f:43675 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.lastflushedseqids 2024-11-08T01:47:01,300 INFO [RS:0;8118b061437f:42871 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T01:47:01,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:01,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42871-0x101181fee630001, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:01,300 INFO [RS:0;8118b061437f:42871 {}] regionserver.HRegionServer(1031): Exiting; stopping=8118b061437f,42871,1731030415338; zookeeper connection closed. 2024-11-08T01:47:01,301 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c963d2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c963d2 2024-11-08T01:47:01,301 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-08T01:47:01,313 INFO [M:0;8118b061437f:43675 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T01:47:01,313 INFO [M:0;8118b061437f:43675 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T01:47:01,313 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T01:47:01,313 INFO [M:0;8118b061437f:43675 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,313 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,313 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T01:47:01,313 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,314 INFO [M:0;8118b061437f:43675 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-08T01:47:01,322 DEBUG [M:0;8118b061437f:43675 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f38e2c140e0a48b381e6cb4f6416df9d 2024-11-08T01:47:01,335 DEBUG [M:0;8118b061437f:43675 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f38e2c140e0a48b381e6cb4f6416df9d is 82, key is hbase:meta,,1/info:regioninfo/1731030417934/Put/seqid=0 2024-11-08T01:47:01,336 INFO [M:0;8118b061437f:43675 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f38e2c140e0a48b381e6cb4f6416df9d 2024-11-08T01:47:01,348 DEBUG [M:0;8118b061437f:43675 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 723e3d83548b4914985b12ee11bc4910 2024-11-08T01:47:01,362 DEBUG [M:0;8118b061437f:43675 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/723e3d83548b4914985b12ee11bc4910 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731030418952/Put/seqid=0 2024-11-08T01:47:01,363 INFO [M:0;8118b061437f:43675 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/723e3d83548b4914985b12ee11bc4910 2024-11-08T01:47:01,374 DEBUG [M:0;8118b061437f:43675 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d7bd766341648e6a7b9d421923950e0 2024-11-08T01:47:01,391 DEBUG [M:0;8118b061437f:43675 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4d7bd766341648e6a7b9d421923950e0 is 69, key is 8118b061437f,42871,1731030415338/rs:state/1731030417043/Put/seqid=0 2024-11-08T01:47:01,392 INFO [M:0;8118b061437f:43675 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4d7bd766341648e6a7b9d421923950e0 2024-11-08T01:47:01,394 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f38e2c140e0a48b381e6cb4f6416df9d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f38e2c140e0a48b381e6cb4f6416df9d 2024-11-08T01:47:01,395 INFO [M:0;8118b061437f:43675 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f38e2c140e0a48b381e6cb4f6416df9d, entries=8, sequenceid=84, filesize=5.5 K 2024-11-08T01:47:01,396 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/723e3d83548b4914985b12ee11bc4910 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/723e3d83548b4914985b12ee11bc4910 2024-11-08T01:47:01,397 INFO [M:0;8118b061437f:43675 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/723e3d83548b4914985b12ee11bc4910, entries=9, sequenceid=84, filesize=6.5 K 2024-11-08T01:47:01,398 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4d7bd766341648e6a7b9d421923950e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4d7bd766341648e6a7b9d421923950e0 2024-11-08T01:47:01,400 INFO [M:0;8118b061437f:43675 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4d7bd766341648e6a7b9d421923950e0, entries=2, sequenceid=84, filesize=5.1 K 2024-11-08T01:47:01,401 INFO [M:0;8118b061437f:43675 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false 2024-11-08T01:47:01,402 INFO [M:0;8118b061437f:43675 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,403 DEBUG [M:0;8118b061437f:43675 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731030421313Disabling compacts and flushes for region at 1731030421313Disabling writes for close at 1731030421313Obtaining lock to block concurrent updates at 1731030421314 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731030421314Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1731030421314Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731030421314Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731030421315 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731030421335 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731030421335Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731030421339 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731030421361 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731030421362 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731030421366 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731030421391 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731030421391Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1731030421394 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1731030421396 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1731030421397 (+1 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false at 1731030421401 (+4 ms)Writing region close event to WAL at 1731030421402 (+1 ms)Closed at 1731030421402 2024-11-08T01:47:01,404 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T01:47:01,404 INFO [M:0;8118b061437f:43675 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T01:47:01,404 INFO [M:0;8118b061437f:43675 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43675 2024-11-08T01:47:01,405 INFO [M:0;8118b061437f:43675 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T01:47:01,514 INFO [M:0;8118b061437f:43675 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T01:47:01,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:01,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43675-0x101181fee630000, quorum=127.0.0.1:50081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:01,519 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T01:47:01,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T01:47:01,539 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,546 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: master/8118b061437f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/8118b061437f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731030420857 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731030421086 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/8118b061437f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731030421078 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/8118b061437f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731030419303 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 463), ProcessCount=11 (was 11), AvailableMemoryMB=7911 (was 7103) - AvailableMemoryMB LEAK? - 2024-11-08T01:47:01,552 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=7910 2024-11-08T01:47:01,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/hadoop-log-dir so I do NOT create it in target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03 2024-11-08T01:47:01,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/hadoop-tmp-dir so I do NOT create it in target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03 2024-11-08T01:47:01,553 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,555 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/cluster_f6e26bc7-1626-8e9e-32a8-46f9bd5f1ee5, deleteOnExit=true 2024-11-08T01:47:01,559 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/cluster_f6e26bc7-1626-8e9e-32a8-46f9bd5f1ee5/zookeeper_0, clientPort=49616, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/cluster_f6e26bc7-1626-8e9e-32a8-46f9bd5f1ee5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/cluster_f6e26bc7-1626-8e9e-32a8-46f9bd5f1ee5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T01:47:01,560 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49616 2024-11-08T01:47:01,570 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 with version=8 2024-11-08T01:47:01,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/hbase-staging 2024-11-08T01:47:01,573 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8118b061437f:0 server-side Connection retries=45 2024-11-08T01:47:01,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,573 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T01:47:01,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T01:47:01,573 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T01:47:01,574 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T01:47:01,574 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44413 2024-11-08T01:47:01,576 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44413 connecting to ZooKeeper ensemble=127.0.0.1:49616 2024-11-08T01:47:01,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444130x0, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T01:47:01,629 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44413-0x10118200be10000 connected 2024-11-08T01:47:01,723 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,724 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:01,728 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7, hbase.cluster.distributed=false 2024-11-08T01:47:01,733 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T01:47:01,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44413 2024-11-08T01:47:01,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44413 2024-11-08T01:47:01,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44413 2024-11-08T01:47:01,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44413 2024-11-08T01:47:01,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44413 2024-11-08T01:47:01,753 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8118b061437f:0 server-side Connection retries=45 2024-11-08T01:47:01,753 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,753 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,753 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T01:47:01,753 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,753 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T01:47:01,753 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T01:47:01,753 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T01:47:01,754 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44791 2024-11-08T01:47:01,755 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44791 connecting to ZooKeeper ensemble=127.0.0.1:49616 2024-11-08T01:47:01,756 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,756 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,764 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447910x0, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T01:47:01,765 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44791-0x10118200be10001 connected 2024-11-08T01:47:01,765 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:01,765 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T01:47:01,766 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.cache 2024-11-08T01:47:01,766 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T01:47:01,766 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T01:47:01,766 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T01:47:01,767 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T01:47:01,767 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T01:47:01,767 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T01:47:01,767 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T01:47:01,767 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T01:47:01,767 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence is complete. 2024-11-08T01:47:01,768 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T01:47:01,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T01:47:01,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44791 2024-11-08T01:47:01,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44791 2024-11-08T01:47:01,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44791 2024-11-08T01:47:01,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44791 2024-11-08T01:47:01,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44791 2024-11-08T01:47:01,786 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8118b061437f:0 server-side Connection retries=45 2024-11-08T01:47:01,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,786 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T01:47:01,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:01,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T01:47:01,787 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T01:47:01,787 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T01:47:01,787 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43839 2024-11-08T01:47:01,789 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43839 connecting to ZooKeeper ensemble=127.0.0.1:49616 2024-11-08T01:47:01,789 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,789 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438390x0, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T01:47:01,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:438390x0, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:01,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43839-0x10118200be10002 connected 2024-11-08T01:47:01,804 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T01:47:01,805 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.cache 2024-11-08T01:47:01,805 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T01:47:01,805 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T01:47:01,805 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T01:47:01,806 INFO [Thread-121 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T01:47:01,806 WARN [Thread-121 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T01:47:01,806 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T01:47:01,806 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T01:47:01,806 INFO [Thread-121 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence is complete. 2024-11-08T01:47:01,806 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T01:47:01,807 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T01:47:01,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T01:47:01,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43839 2024-11-08T01:47:01,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43839 2024-11-08T01:47:01,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43839 2024-11-08T01:47:01,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43839 2024-11-08T01:47:01,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43839 2024-11-08T01:47:01,829 DEBUG [M:0;8118b061437f:44413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8118b061437f:44413 2024-11-08T01:47:01,829 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8118b061437f,44413,1731030421572 2024-11-08T01:47:01,838 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:47:01,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:47:01,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:47:01,839 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8118b061437f,44413,1731030421572 2024-11-08T01:47:01,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T01:47:01,848 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T01:47:01,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:01,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:01,849 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:01,849 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T01:47:01,850 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8118b061437f,44413,1731030421572 from backup master directory 2024-11-08T01:47:01,859 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:47:01,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:47:01,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8118b061437f,44413,1731030421572 2024-11-08T01:47:01,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T01:47:01,859 WARN [master/8118b061437f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T01:47:01,859 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8118b061437f,44413,1731030421572 2024-11-08T01:47:01,860 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/hbase.id] with ID: 406e57db-4cc4-49a0-911e-1dc2177b2b5a 2024-11-08T01:47:01,861 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.tmp/hbase.id 2024-11-08T01:47:01,864 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/hbase.id] 2024-11-08T01:47:01,894 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:01,895 INFO [master/8118b061437f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T01:47:01,897 INFO [master/8118b061437f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-08T01:47:01,908 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:01,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:01,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:01,913 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T01:47:01,914 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T01:47:01,915 INFO [master/8118b061437f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:47:01,932 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store 2024-11-08T01:47:01,937 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:01,937 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T01:47:01,937 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,937 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,937 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T01:47:01,937 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,937 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:01,937 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731030421937Disabling compacts and flushes for region at 1731030421937Disabling writes for close at 1731030421937Writing region close event to WAL at 1731030421937Closed at 1731030421937 2024-11-08T01:47:01,938 WARN [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/.initializing 2024-11-08T01:47:01,938 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/WALs/8118b061437f,44413,1731030421572 2024-11-08T01:47:01,942 INFO [master/8118b061437f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T01:47:01,945 INFO [master/8118b061437f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8118b061437f%2C44413%2C1731030421572, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/WALs/8118b061437f,44413,1731030421572, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/oldWALs, maxLogs=10 2024-11-08T01:47:01,965 INFO [master/8118b061437f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/WALs/8118b061437f,44413,1731030421572/8118b061437f%2C44413%2C1731030421572.1731030421946 2024-11-08T01:47:01,965 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:47:01,965 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:47:01,965 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:01,965 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,966 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T01:47:01,971 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:01,971 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:01,971 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T01:47:01,974 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:01,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:47:01,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,977 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T01:47:01,977 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:01,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:47:01,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T01:47:01,981 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:01,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:47:01,981 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,981 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,982 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,982 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,982 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,982 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T01:47:01,982 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T01:47:01,988 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T01:47:01,988 INFO [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66315722, jitterRate=-0.011818736791610718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T01:47:01,989 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731030421966Initializing all the Stores at 1731030421966Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030421966Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030421966Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030421966Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030421966Cleaning up temporary data from old regions at 1731030421982 (+16 ms)Region opened successfully at 1731030421989 (+7 ms) 2024-11-08T01:47:01,990 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T01:47:01,996 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f83140a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8118b061437f/172.17.0.2:0 2024-11-08T01:47:01,997 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T01:47:01,997 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T01:47:01,997 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T01:47:01,998 INFO [master/8118b061437f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T01:47:01,998 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T01:47:01,998 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-08T01:47:01,998 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T01:47:02,000 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T01:47:02,001 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T01:47:02,102 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T01:47:02,103 INFO [master/8118b061437f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T01:47:02,106 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T01:47:02,195 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T01:47:02,196 INFO [master/8118b061437f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T01:47:02,199 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T01:47:02,213 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T01:47:02,216 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T01:47:02,228 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T01:47:02,234 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T01:47:02,244 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T01:47:02,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:02,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:02,255 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:02,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,255 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,256 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8118b061437f,44413,1731030421572, sessionid=0x10118200be10000, setting cluster-up flag (Was=false) 2024-11-08T01:47:02,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,280 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,312 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T01:47:02,313 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8118b061437f,44413,1731030421572 2024-11-08T01:47:02,333 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,435 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T01:47:02,439 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8118b061437f,44413,1731030421572 2024-11-08T01:47:02,441 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T01:47:02,445 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T01:47:02,445 INFO [master/8118b061437f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T01:47:02,445 INFO [master/8118b061437f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T01:47:02,446 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8118b061437f,44413,1731030421572 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T01:47:02,448 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:47:02,448 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:47:02,448 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:47:02,448 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8118b061437f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T01:47:02,448 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8118b061437f:0, corePoolSize=10, maxPoolSize=10 2024-11-08T01:47:02,448 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,449 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8118b061437f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T01:47:02,449 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,450 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731030452450 2024-11-08T01:47:02,450 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T01:47:02,450 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T01:47:02,450 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T01:47:02,450 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T01:47:02,450 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T01:47:02,450 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T01:47:02,450 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T01:47:02,451 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T01:47:02,451 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,451 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:02,451 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T01:47:02,452 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T01:47:02,452 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T01:47:02,451 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T01:47:02,452 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T01:47:02,452 INFO [master/8118b061437f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T01:47:02,453 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.large.0-1731030422452,5,FailOnTimeoutGroup] 2024-11-08T01:47:02,460 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.small.0-1731030422453,5,FailOnTimeoutGroup] 2024-11-08T01:47:02,460 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,460 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T01:47:02,460 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,460 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,469 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T01:47:02,469 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:47:02,476 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:02,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T01:47:02,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T01:47:02,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:02,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:02,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T01:47:02,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T01:47:02,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:02,483 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:02,483 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T01:47:02,486 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T01:47:02,486 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:02,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:02,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T01:47:02,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T01:47:02,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:02,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:02,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T01:47:02,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:47:02,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:47:02,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T01:47:02,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T01:47:02,491 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T01:47:02,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T01:47:02,496 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T01:47:02,496 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66159254, jitterRate=-0.014150291681289673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T01:47:02,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731030422476Initializing all the Stores at 1731030422476Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030422476Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030422477 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030422477Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030422477Cleaning up temporary data from old regions at 1731030422491 (+14 ms)Region opened successfully at 1731030422497 (+6 ms) 2024-11-08T01:47:02,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T01:47:02,497 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T01:47:02,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T01:47:02,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T01:47:02,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T01:47:02,498 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T01:47:02,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731030422497Disabling compacts and flushes for region at 1731030422497Disabling writes for close at 1731030422498 (+1 ms)Writing region close event to WAL at 1731030422498Closed at 1731030422498 2024-11-08T01:47:02,499 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T01:47:02,499 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T01:47:02,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T01:47:02,501 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T01:47:02,502 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T01:47:02,520 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(746): ClusterId : 406e57db-4cc4-49a0-911e-1dc2177b2b5a 2024-11-08T01:47:02,520 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(746): ClusterId : 406e57db-4cc4-49a0-911e-1dc2177b2b5a 2024-11-08T01:47:02,521 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T01:47:02,521 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T01:47:02,594 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T01:47:02,594 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T01:47:02,594 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T01:47:02,594 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T01:47:02,646 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T01:47:02,646 DEBUG [RS:1;8118b061437f:43839 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bbfcf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8118b061437f/172.17.0.2:0 2024-11-08T01:47:02,647 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T01:47:02,648 DEBUG [RS:0;8118b061437f:44791 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f70af9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8118b061437f/172.17.0.2:0 2024-11-08T01:47:02,652 WARN [8118b061437f:44413 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T01:47:02,663 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;8118b061437f:43839 2024-11-08T01:47:02,663 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8118b061437f:44791 2024-11-08T01:47:02,663 INFO [RS:0;8118b061437f:44791 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T01:47:02,663 INFO [RS:0;8118b061437f:44791 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T01:47:02,663 INFO [RS:1;8118b061437f:43839 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T01:47:02,663 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T01:47:02,663 INFO [RS:1;8118b061437f:43839 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T01:47:02,663 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T01:47:02,664 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(2659): reportForDuty to master=8118b061437f,44413,1731030421572 with port=43839, startcode=1731030421786 2024-11-08T01:47:02,664 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(2659): reportForDuty to master=8118b061437f,44413,1731030421572 with port=44791, startcode=1731030421752 2024-11-08T01:47:02,664 DEBUG [RS:0;8118b061437f:44791 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T01:47:02,664 DEBUG [RS:1;8118b061437f:43839 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T01:47:02,667 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T01:47:02,667 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42305, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T01:47:02,667 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44413 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8118b061437f,44791,1731030421752 2024-11-08T01:47:02,667 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44413 {}] master.ServerManager(517): Registering regionserver=8118b061437f,44791,1731030421752 2024-11-08T01:47:02,669 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44413 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8118b061437f,43839,1731030421786 2024-11-08T01:47:02,669 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44413 {}] master.ServerManager(517): Registering regionserver=8118b061437f,43839,1731030421786 2024-11-08T01:47:02,669 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:47:02,669 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T01:47:02,669 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T01:47:02,670 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:47:02,670 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T01:47:02,670 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T01:47:02,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T01:47:02,715 DEBUG [RS:0;8118b061437f:44791 {}] zookeeper.ZKUtil(111): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8118b061437f,44791,1731030421752 2024-11-08T01:47:02,715 WARN [RS:0;8118b061437f:44791 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T01:47:02,715 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8118b061437f,43839,1731030421786] 2024-11-08T01:47:02,715 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8118b061437f,44791,1731030421752] 2024-11-08T01:47:02,715 INFO [RS:0;8118b061437f:44791 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:47:02,716 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752 2024-11-08T01:47:02,716 DEBUG [RS:1;8118b061437f:43839 {}] zookeeper.ZKUtil(111): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8118b061437f,43839,1731030421786 2024-11-08T01:47:02,716 WARN [RS:1;8118b061437f:43839 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T01:47:02,716 INFO [RS:1;8118b061437f:43839 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:47:02,716 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43839,1731030421786 2024-11-08T01:47:02,722 INFO [RS:0;8118b061437f:44791 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T01:47:02,723 INFO [RS:1;8118b061437f:43839 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T01:47:02,727 INFO [RS:0;8118b061437f:44791 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T01:47:02,728 INFO [RS:0;8118b061437f:44791 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T01:47:02,728 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,728 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T01:47:02,728 INFO [RS:0;8118b061437f:44791 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T01:47:02,729 INFO [RS:1;8118b061437f:43839 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T01:47:02,729 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,729 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,729 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,729 INFO [RS:1;8118b061437f:43839 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T01:47:02,729 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,729 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,729 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,729 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T01:47:02,729 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8118b061437f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 INFO [RS:1;8118b061437f:43839 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:47:02,730 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,730 DEBUG [RS:0;8118b061437f:44791 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:47:02,730 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8118b061437f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T01:47:02,730 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,731 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,731 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,731 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,731 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,731 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:02,731 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,731 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:47:02,731 DEBUG [RS:1;8118b061437f:43839 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:47:02,731 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,731 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,731 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,731 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,731 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44791,1731030421752-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T01:47:02,731 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,732 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,732 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,732 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,732 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,732 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43839,1731030421786-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T01:47:02,746 INFO [RS:0;8118b061437f:44791 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T01:47:02,746 INFO [RS:1;8118b061437f:43839 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T01:47:02,746 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,43839,1731030421786-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,746 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44791,1731030421752-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,746 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,746 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,746 INFO [RS:1;8118b061437f:43839 {}] regionserver.Replication(171): 8118b061437f,43839,1731030421786 started 2024-11-08T01:47:02,746 INFO [RS:0;8118b061437f:44791 {}] regionserver.Replication(171): 8118b061437f,44791,1731030421752 started 2024-11-08T01:47:02,759 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,759 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:02,760 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1482): Serving as 8118b061437f,43839,1731030421786, RpcServer on 8118b061437f/172.17.0.2:43839, sessionid=0x10118200be10002 2024-11-08T01:47:02,760 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1482): Serving as 8118b061437f,44791,1731030421752, RpcServer on 8118b061437f/172.17.0.2:44791, sessionid=0x10118200be10001 2024-11-08T01:47:02,760 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T01:47:02,760 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T01:47:02,760 DEBUG [RS:0;8118b061437f:44791 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8118b061437f,44791,1731030421752 2024-11-08T01:47:02,760 DEBUG [RS:1;8118b061437f:43839 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8118b061437f,43839,1731030421786 2024-11-08T01:47:02,760 DEBUG [RS:0;8118b061437f:44791 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,44791,1731030421752' 2024-11-08T01:47:02,760 DEBUG [RS:1;8118b061437f:43839 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,43839,1731030421786' 2024-11-08T01:47:02,760 DEBUG [RS:1;8118b061437f:43839 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T01:47:02,760 DEBUG [RS:0;8118b061437f:44791 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T01:47:02,760 DEBUG [RS:1;8118b061437f:43839 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T01:47:02,761 DEBUG [RS:0;8118b061437f:44791 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T01:47:02,761 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T01:47:02,761 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T01:47:02,761 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T01:47:02,761 DEBUG [RS:1;8118b061437f:43839 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8118b061437f,43839,1731030421786 2024-11-08T01:47:02,761 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T01:47:02,761 DEBUG [RS:1;8118b061437f:43839 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,43839,1731030421786' 2024-11-08T01:47:02,761 DEBUG [RS:0;8118b061437f:44791 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8118b061437f,44791,1731030421752 2024-11-08T01:47:02,761 DEBUG [RS:1;8118b061437f:43839 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T01:47:02,761 DEBUG [RS:0;8118b061437f:44791 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,44791,1731030421752' 2024-11-08T01:47:02,761 DEBUG [RS:0;8118b061437f:44791 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T01:47:02,761 DEBUG [RS:1;8118b061437f:43839 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T01:47:02,761 DEBUG [RS:0;8118b061437f:44791 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T01:47:02,762 DEBUG [RS:1;8118b061437f:43839 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T01:47:02,762 DEBUG [RS:0;8118b061437f:44791 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T01:47:02,762 INFO [RS:1;8118b061437f:43839 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T01:47:02,762 INFO [RS:0;8118b061437f:44791 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T01:47:02,762 INFO [RS:1;8118b061437f:43839 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T01:47:02,762 INFO [RS:0;8118b061437f:44791 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T01:47:02,863 INFO [RS:1;8118b061437f:43839 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T01:47:02,863 INFO [RS:0;8118b061437f:44791 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T01:47:02,864 INFO [RS:1;8118b061437f:43839 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C43839%2C1731030421786, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43839,1731030421786, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:47:02,864 INFO [RS:0;8118b061437f:44791 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C44791%2C1731030421752, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:47:02,885 INFO [RS:0;8118b061437f:44791 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752/8118b061437f%2C44791%2C1731030421752.1731030422868 2024-11-08T01:47:02,885 INFO [RS:1;8118b061437f:43839 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43839,1731030421786/8118b061437f%2C43839%2C1731030421786.1731030422868 2024-11-08T01:47:02,885 DEBUG [RS:0;8118b061437f:44791 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:47:02,885 DEBUG [RS:1;8118b061437f:43839 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:47:02,902 DEBUG [8118b061437f:44413 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T01:47:02,903 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(204): Hosts are {8118b061437f=0} racks are {/default-rack=0} 2024-11-08T01:47:02,903 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T01:47:02,904 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T01:47:02,904 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T01:47:02,904 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T01:47:02,904 INFO [8118b061437f:44413 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T01:47:02,904 INFO [8118b061437f:44413 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T01:47:02,904 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T01:47:02,904 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8118b061437f,44791,1731030421752 2024-11-08T01:47:02,905 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8118b061437f,44791,1731030421752, state=OPENING 2024-11-08T01:47:02,922 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T01:47:02,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,933 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:02,934 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T01:47:02,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:02,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:02,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:02,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8118b061437f,44791,1731030421752}] 2024-11-08T01:47:03,089 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T01:47:03,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48003, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T01:47:03,095 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T01:47:03,095 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:47:03,095 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T01:47:03,096 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C44791%2C1731030421752.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:47:03,117 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752/8118b061437f%2C44791%2C1731030421752.meta.1731030423097.meta 2024-11-08T01:47:03,117 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:47:03,117 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:47:03,117 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T01:47:03,118 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T01:47:03,118 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T01:47:03,118 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T01:47:03,118 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:03,118 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T01:47:03,118 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T01:47:03,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T01:47:03,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T01:47:03,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:03,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:03,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T01:47:03,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T01:47:03,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:03,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:03,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T01:47:03,122 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T01:47:03,122 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:03,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:03,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T01:47:03,122 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T01:47:03,122 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:03,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:03,123 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T01:47:03,123 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:47:03,124 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:47:03,124 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T01:47:03,124 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T01:47:03,124 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T01:47:03,125 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T01:47:03,125 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64168665, jitterRate=-0.043812379240989685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T01:47:03,125 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T01:47:03,126 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731030423118Writing region info on filesystem at 1731030423118Initializing all the Stores at 1731030423118Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030423118Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030423119 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030423119Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030423119Cleaning up temporary data from old regions at 1731030423124 (+5 ms)Running coprocessor post-open hooks at 1731030423125 (+1 ms)Region opened successfully at 1731030423126 (+1 ms) 2024-11-08T01:47:03,126 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731030423089 2024-11-08T01:47:03,129 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T01:47:03,129 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T01:47:03,129 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8118b061437f,44791,1731030421752 2024-11-08T01:47:03,130 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8118b061437f,44791,1731030421752, state=OPEN 2024-11-08T01:47:03,161 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:03,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:03,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:03,161 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8118b061437f,44791,1731030421752 2024-11-08T01:47:03,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:03,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:03,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:03,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T01:47:03,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8118b061437f,44791,1731030421752 in 227 msec 2024-11-08T01:47:03,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T01:47:03,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 666 msec 2024-11-08T01:47:03,169 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T01:47:03,169 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T01:47:03,172 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T01:47:03,172 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8118b061437f,44791,1731030421752, seqNum=-1] 2024-11-08T01:47:03,172 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T01:47:03,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48925, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T01:47:03,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 735 msec 2024-11-08T01:47:03,180 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731030423180, completionTime=-1 2024-11-08T01:47:03,180 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T01:47:03,180 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T01:47:03,183 INFO [master/8118b061437f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T01:47:03,183 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731030483183 2024-11-08T01:47:03,183 INFO [master/8118b061437f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731030543183 2024-11-08T01:47:03,183 INFO [master/8118b061437f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-08T01:47:03,183 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44413,1731030421572-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:03,184 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44413,1731030421572-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:03,184 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44413,1731030421572-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:03,184 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8118b061437f:44413, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:03,184 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:03,184 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:03,187 DEBUG [master/8118b061437f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T01:47:03,189 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.330sec 2024-11-08T01:47:03,190 INFO [master/8118b061437f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T01:47:03,190 INFO [master/8118b061437f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T01:47:03,190 INFO [master/8118b061437f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T01:47:03,190 INFO [master/8118b061437f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T01:47:03,190 INFO [master/8118b061437f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T01:47:03,190 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44413,1731030421572-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T01:47:03,190 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44413,1731030421572-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T01:47:03,191 DEBUG [master/8118b061437f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T01:47:03,191 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T01:47:03,191 INFO [master/8118b061437f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,44413,1731030421572-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:03,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T01:47:03,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8118b061437f,44413,-1 for getting cluster id 2024-11-08T01:47:03,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T01:47:03,223 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '406e57db-4cc4-49a0-911e-1dc2177b2b5a' 2024-11-08T01:47:03,224 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T01:47:03,224 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "406e57db-4cc4-49a0-911e-1dc2177b2b5a" 2024-11-08T01:47:03,225 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@157ae13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T01:47:03,225 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8118b061437f,44413,-1] 2024-11-08T01:47:03,225 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T01:47:03,226 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:03,228 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T01:47:03,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T01:47:03,231 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T01:47:03,233 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8118b061437f,44791,1731030421752, seqNum=-1] 2024-11-08T01:47:03,234 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T01:47:03,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T01:47:03,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8118b061437f,44413,1731030421572 2024-11-08T01:47:03,239 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T01:47:03,241 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8118b061437f,44413,1731030421572 2024-11-08T01:47:03,241 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b3ad739 2024-11-08T01:47:03,241 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T01:47:03,243 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T01:47:03,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731030423239', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T01:47:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 2024-11-08T01:47:03,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T01:47:03,247 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:03,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731030423239" procId is: 4 2024-11-08T01:47:03,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T01:47:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:47:03,258 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4598fca782bf34f9926ed286f64ea7bf, NAME => 'testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731030423239', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:47:03,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:03,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(1722): Closing 4598fca782bf34f9926ed286f64ea7bf, disabling compactions & flushes 2024-11-08T01:47:03,264 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:03,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:03,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. after waiting 0 ms 2024-11-08T01:47:03,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:03,264 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:03,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731030423239-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4598fca782bf34f9926ed286f64ea7bf: Waiting for close lock at 1731030423264Disabling compacts and flushes for region at 1731030423264Disabling writes for close at 1731030423264Writing region close event to WAL at 1731030423264Closed at 1731030423264 2024-11-08T01:47:03,265 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T01:47:03,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731030423265"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731030423265"}]},"ts":"1731030423265"} 2024-11-08T01:47:03,268 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T01:47:03,269 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T01:47:03,270 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731030423239","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731030423269"}]},"ts":"1731030423269"} 2024-11-08T01:47:03,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731030423239, state=ENABLING in hbase:meta 2024-11-08T01:47:03,272 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {8118b061437f=0} racks are {/default-rack=0} 2024-11-08T01:47:03,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T01:47:03,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T01:47:03,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T01:47:03,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T01:47:03,273 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T01:47:03,273 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T01:47:03,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T01:47:03,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN}] 2024-11-08T01:47:03,275 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN 2024-11-08T01:47:03,275 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN; state=OFFLINE, location=8118b061437f,44791,1731030421752; forceNewPlan=false, retain=false 2024-11-08T01:47:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:47:03,397 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T01:47:03,426 INFO [8118b061437f:44413 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T01:47:03,426 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4598fca782bf34f9926ed286f64ea7bf, regionState=OPENING, regionLocation=8118b061437f,44791,1731030421752 2024-11-08T01:47:03,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN because future has completed 2024-11-08T01:47:03,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4598fca782bf34f9926ed286f64ea7bf, server=8118b061437f,44791,1731030421752}] 2024-11-08T01:47:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:47:03,585 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:03,586 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4598fca782bf34f9926ed286f64ea7bf, NAME => 'testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:47:03,587 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731030423239 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,587 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:03,587 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,588 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,590 INFO [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,596 INFO [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4598fca782bf34f9926ed286f64ea7bf columnFamilyName family 2024-11-08T01:47:03,596 DEBUG [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:03,597 INFO [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] regionserver.HStore(327): Store=4598fca782bf34f9926ed286f64ea7bf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:47:03,597 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,598 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,598 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,598 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,598 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,599 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,606 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T01:47:03,606 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4598fca782bf34f9926ed286f64ea7bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61491743, jitterRate=-0.08370162546634674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T01:47:03,606 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:03,608 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4598fca782bf34f9926ed286f64ea7bf: Running coprocessor pre-open hook at 1731030423588Writing region info on filesystem at 1731030423588Initializing all the Stores at 1731030423589 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030423589Cleaning up temporary data from old regions at 1731030423598 (+9 ms)Running coprocessor post-open hooks at 1731030423606 (+8 ms)Region opened successfully at 1731030423608 (+2 ms) 2024-11-08T01:47:03,609 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf., pid=6, masterSystemTime=1731030423581 2024-11-08T01:47:03,613 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4598fca782bf34f9926ed286f64ea7bf, regionState=OPEN, openSeqNum=2, regionLocation=8118b061437f,44791,1731030421752 2024-11-08T01:47:03,613 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:03,613 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:03,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4598fca782bf34f9926ed286f64ea7bf, server=8118b061437f,44791,1731030421752 because future has completed 2024-11-08T01:47:03,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T01:47:03,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4598fca782bf34f9926ed286f64ea7bf, server=8118b061437f,44791,1731030421752 in 188 msec 2024-11-08T01:47:03,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T01:47:03,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN in 347 msec 2024-11-08T01:47:03,623 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T01:47:03,623 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731030423239","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731030423623"}]},"ts":"1731030423623"} 2024-11-08T01:47:03,625 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731030423239, state=ENABLED in hbase:meta 2024-11-08T01:47:03,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T01:47:03,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731030423239 in 381 msec 2024-11-08T01:47:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T01:47:03,878 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731030423239 completed 2024-11-08T01:47:03,878 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731030423239 get assigned. Timeout = 60000ms 2024-11-08T01:47:03,878 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T01:47:03,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731030423239 assigned to meta. Checking AM states. 2024-11-08T01:47:03,883 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T01:47:03,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731030423239 assigned. 2024-11-08T01:47:03,890 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731030423239', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf., hostname=8118b061437f,44791,1731030421752, seqNum=2] 2024-11-08T01:47:03,895 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4598fca782bf34f9926ed286f64ea7bf 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T01:47:03,904 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07f0e4db5797446aa2d005de4929848e 2024-11-08T01:47:03,920 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/.tmp/family/07f0e4db5797446aa2d005de4929848e is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T01:47:03,920 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/.tmp/family/07f0e4db5797446aa2d005de4929848e 2024-11-08T01:47:03,922 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/.tmp/family/07f0e4db5797446aa2d005de4929848e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e 2024-11-08T01:47:03,923 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e, delay=988 ms 2024-11-08T01:47:03,924 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T01:47:03,925 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 4598fca782bf34f9926ed286f64ea7bf in 30ms, sequenceid=6, compaction requested=false 2024-11-08T01:47:03,925 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731030423239' 2024-11-08T01:47:03,926 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4598fca782bf34f9926ed286f64ea7bf: 2024-11-08T01:47:04,854 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T01:47:04,854 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T01:47:04,861 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731030423239 2024-11-08T01:47:04,861 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731030423239 Metrics about Tables on a single HBase RegionServer 2024-11-08T01:47:04,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T01:47:04,864 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T01:47:04,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T01:47:04,864 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T01:47:04,913 DEBUG [hfile-prefetch-1731030420857 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 07f0e4db5797446aa2d005de4929848e_0. 2024-11-08T01:47:04,913 DEBUG [hfile-prefetch-1731030420857 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 07f0e4db5797446aa2d005de4929848e_117. 2024-11-08T01:47:04,914 DEBUG [hfile-prefetch-1731030420857 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e, with total blocks 2, and data blocks 1 2024-11-08T01:47:04,914 DEBUG [hfile-prefetch-1731030420857 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T01:47:04,914 DEBUG [hfile-prefetch-1731030420857 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T01:47:04,914 DEBUG [hfile-prefetch-1731030420857 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e. Will sleep 100ms and try the verification again. 2024-11-08T01:47:04,926 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;8118b061437f:44791,5,FailOnTimeoutGroup] 2024-11-08T01:47:04,926 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8118b061437f,44791,1731030421752' ***** 2024-11-08T01:47:04,927 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-08T01:47:04,927 INFO [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T01:47:04,934 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T01:47:04,934 INFO [regionserver/8118b061437f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:04,992 INFO [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence 2024-11-08T01:47:04,992 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T01:47:04,992 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T01:47:04,992 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T01:47:04,992 INFO [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T01:47:04,992 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T01:47:05,002 DEBUG [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1488): Thread RS:0;8118b061437f:44791 started persisting bucket cache to file 2024-11-08T01:47:05,003 DEBUG [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:05,003 DEBUG [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:05,009 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T01:47:05,009 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T01:47:05,009 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:05,009 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:05,010 DEBUG [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T01:47:05,011 DEBUG [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731030425002 2024-11-08T01:47:05,011 DEBUG [RS:0;8118b061437f:44791 {}] bucket.BucketCache(1510): Thread RS:0;8118b061437f:44791 finished persisting bucket cache to file, renaming 2024-11-08T01:47:05,011 INFO [RS:0;8118b061437f:44791 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T01:47:05,012 INFO [RS:0;8118b061437f:44791 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T01:47:05,012 INFO [RS:0;8118b061437f:44791 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T01:47:05,012 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(3091): Received CLOSE for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:05,012 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(959): stopping server 8118b061437f,44791,1731030421752 2024-11-08T01:47:05,012 INFO [RS:0;8118b061437f:44791 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T01:47:05,012 INFO [RS:0;8118b061437f:44791 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8118b061437f:44791. 2024-11-08T01:47:05,012 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T01:47:05,013 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4598fca782bf34f9926ed286f64ea7bf, disabling compactions & flushes 2024-11-08T01:47:05,013 DEBUG [RS:0;8118b061437f:44791 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:05,013 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:05,013 DEBUG [RS:0;8118b061437f:44791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:05,013 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:05,013 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. after waiting 0 ms 2024-11-08T01:47:05,013 INFO [RS:0;8118b061437f:44791 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T01:47:05,013 INFO [RS:0;8118b061437f:44791 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T01:47:05,013 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:05,013 INFO [RS:0;8118b061437f:44791 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T01:47:05,013 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T01:47:05,014 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T01:47:05,014 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 4598fca782bf34f9926ed286f64ea7bf=testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.} 2024-11-08T01:47:05,014 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T01:47:05,014 DEBUG [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:05,014 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T01:47:05,014 DEBUG [StoreFileCloser-4598fca782bf34f9926ed286f64ea7bf-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e 2024-11-08T01:47:05,014 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T01:47:05,014 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T01:47:05,014 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T01:47:05,014 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T01:47:05,014 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T01:47:05,014 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731030425009 2024-11-08T01:47:05,014 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T01:47:05,014 WARN [hfile-prefetch-1731030420857 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-08T01:47:05,014 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-08T01:47:05,015 DEBUG [hfile-prefetch-1731030420857 {}] hfile.PrefetchExecutor(121): Prefetch completed for 07f0e4db5797446aa2d005de4929848e 2024-11-08T01:47:05,019 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T01:47:05,019 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:05,019 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4598fca782bf34f9926ed286f64ea7bf: Waiting for close lock at 1731030425012Running coprocessor pre-close hooks at 1731030425012Disabling compacts and flushes for region at 1731030425012Disabling writes for close at 1731030425013 (+1 ms)Writing region close event to WAL at 1731030425014 (+1 ms)Running coprocessor post-close hooks at 1731030425019 (+5 ms)Closed at 1731030425019 2024-11-08T01:47:05,020 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:05,025 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d5d727aa2d1f49c78ea390ea95b95e2a 2024-11-08T01:47:05,037 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/d5d727aa2d1f49c78ea390ea95b95e2a is 209, key is testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf./info:regioninfo/1731030423612/Put/seqid=0 2024-11-08T01:47:05,038 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/d5d727aa2d1f49c78ea390ea95b95e2a 2024-11-08T01:47:05,044 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f55700072e304049b05f4b122e5e6f26 2024-11-08T01:47:05,058 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/ns/f55700072e304049b05f4b122e5e6f26 is 43, key is default/ns:d/1731030423175/Put/seqid=0 2024-11-08T01:47:05,059 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/ns/f55700072e304049b05f4b122e5e6f26 2024-11-08T01:47:05,066 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c11a1826e26f4f078cab664f5dfb1274 2024-11-08T01:47:05,081 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/table/c11a1826e26f4f078cab664f5dfb1274 is 80, key is testBlockEvictionOnGracefulStop1731030423239/table:state/1731030423623/Put/seqid=0 2024-11-08T01:47:05,081 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/table/c11a1826e26f4f078cab664f5dfb1274 2024-11-08T01:47:05,085 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/d5d727aa2d1f49c78ea390ea95b95e2a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/d5d727aa2d1f49c78ea390ea95b95e2a 2024-11-08T01:47:05,086 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/d5d727aa2d1f49c78ea390ea95b95e2a, delay=940 ms 2024-11-08T01:47:05,087 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/d5d727aa2d1f49c78ea390ea95b95e2a, entries=10, sequenceid=11, filesize=7.0 K 2024-11-08T01:47:05,087 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/ns/f55700072e304049b05f4b122e5e6f26 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/f55700072e304049b05f4b122e5e6f26 2024-11-08T01:47:05,089 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/f55700072e304049b05f4b122e5e6f26, delay=912 ms 2024-11-08T01:47:05,090 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/f55700072e304049b05f4b122e5e6f26, entries=2, sequenceid=11, filesize=5.0 K 2024-11-08T01:47:05,090 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/table/c11a1826e26f4f078cab664f5dfb1274 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/c11a1826e26f4f078cab664f5dfb1274 2024-11-08T01:47:05,091 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/c11a1826e26f4f078cab664f5dfb1274, delay=921 ms 2024-11-08T01:47:05,092 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/c11a1826e26f4f078cab664f5dfb1274, entries=2, sequenceid=11, filesize=5.3 K 2024-11-08T01:47:05,093 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false 2024-11-08T01:47:05,093 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/d5d727aa2d1f49c78ea390ea95b95e2a 2024-11-08T01:47:05,094 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/f55700072e304049b05f4b122e5e6f26 2024-11-08T01:47:05,094 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/c11a1826e26f4f078cab664f5dfb1274 2024-11-08T01:47:05,098 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-08T01:47:05,099 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T01:47:05,099 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T01:47:05,099 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731030425014Running coprocessor pre-close hooks at 1731030425014Disabling compacts and flushes for region at 1731030425014Disabling writes for close at 1731030425014Obtaining lock to block concurrent updates at 1731030425014Preparing flush snapshotting stores in 1588230740 at 1731030425014Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731030425015 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731030425015Flushing 1588230740/info: creating writer at 1731030425015Flushing 1588230740/info: appending metadata at 1731030425037 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731030425037Flushing 1588230740/ns: creating writer at 1731030425039 (+2 ms)Flushing 1588230740/ns: appending metadata at 1731030425058 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731030425058Flushing 1588230740/table: creating writer at 1731030425061 (+3 ms)Flushing 1588230740/table: appending metadata at 1731030425080 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731030425080Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5446480a: reopening flushed file at 1731030425084 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@119b3729: reopening flushed file at 1731030425087 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b0d4a1e: reopening flushed file at 1731030425090 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false at 1731030425093 (+3 ms)Writing region close event to WAL at 1731030425094 (+1 ms)Running coprocessor post-close hooks at 1731030425098 (+4 ms)Closed at 1731030425099 (+1 ms) 2024-11-08T01:47:05,099 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T01:47:05,214 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(976): stopping server 8118b061437f,44791,1731030421752; all regions closed. 2024-11-08T01:47:05,219 DEBUG [RS:0;8118b061437f:44791 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:05,219 INFO [RS:0;8118b061437f:44791 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C44791%2C1731030421752.meta:.meta(num 1731030423097) 2024-11-08T01:47:05,222 DEBUG [RS:0;8118b061437f:44791 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:05,222 INFO [RS:0;8118b061437f:44791 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C44791%2C1731030421752:(num 1731030422868) 2024-11-08T01:47:05,222 DEBUG [RS:0;8118b061437f:44791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:05,222 INFO [RS:0;8118b061437f:44791 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:05,222 INFO [RS:0;8118b061437f:44791 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T01:47:05,223 INFO [RS:0;8118b061437f:44791 {}] hbase.ChoreService(370): Chore service for: regionserver/8118b061437f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T01:47:05,223 INFO [RS:0;8118b061437f:44791 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T01:47:05,223 INFO [regionserver/8118b061437f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T01:47:05,223 INFO [RS:0;8118b061437f:44791 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44791 2024-11-08T01:47:05,248 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8118b061437f,44791,1731030421752 2024-11-08T01:47:05,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T01:47:05,248 INFO [RS:0;8118b061437f:44791 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T01:47:05,259 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8118b061437f,44791,1731030421752] 2024-11-08T01:47:05,269 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8118b061437f,44791,1731030421752 already deleted, retry=false 2024-11-08T01:47:05,270 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 8118b061437f,44791,1731030421752 on 8118b061437f,44413,1731030421572 2024-11-08T01:47:05,278 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 8118b061437f,44791,1731030421752, splitWal=true, meta=true 2024-11-08T01:47:05,281 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 8118b061437f,44791,1731030421752, splitWal=true, meta=true 2024-11-08T01:47:05,281 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 8118b061437f,44791,1731030421752 (carryingMeta=true) 8118b061437f,44791,1731030421752/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2af2ea2c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T01:47:05,282 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 8118b061437f,44791,1731030421752, splitWal=true, meta=true, isMeta: true 2024-11-08T01:47:05,282 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752-splitting 2024-11-08T01:47:05,282 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752-splitting dir is empty, no logs to split. 2024-11-08T01:47:05,282 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8118b061437f,44791,1731030421752 WAL count=0, meta=true 2024-11-08T01:47:05,284 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752-splitting dir is empty, no logs to split. 2024-11-08T01:47:05,284 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8118b061437f,44791,1731030421752 WAL count=0, meta=true 2024-11-08T01:47:05,284 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 8118b061437f,44791,1731030421752 WAL splitting is done? wals=0, meta=true 2024-11-08T01:47:05,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T01:47:05,286 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T01:47:05,287 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T01:47:05,359 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:05,359 INFO [RS:0;8118b061437f:44791 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T01:47:05,359 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44791-0x10118200be10001, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:05,359 INFO [RS:0;8118b061437f:44791 {}] regionserver.HRegionServer(1031): Exiting; stopping=8118b061437f,44791,1731030421752; zookeeper connection closed. 2024-11-08T01:47:05,360 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ec25d9e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ec25d9e 2024-11-08T01:47:05,437 DEBUG [8118b061437f:44413 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T01:47:05,438 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8118b061437f,43839,1731030421786 2024-11-08T01:47:05,440 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8118b061437f,43839,1731030421786, state=OPENING 2024-11-08T01:47:05,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:05,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:05,448 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T01:47:05,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:05,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:05,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8118b061437f,43839,1731030421786}] 2024-11-08T01:47:05,450 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8118b061437f:0 server-side Connection retries=45 2024-11-08T01:47:05,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:05,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:05,450 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T01:47:05,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T01:47:05,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T01:47:05,450 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T01:47:05,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T01:47:05,451 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36769 2024-11-08T01:47:05,453 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36769 connecting to ZooKeeper ensemble=127.0.0.1:49616 2024-11-08T01:47:05,453 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:05,453 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:05,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367690x0, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T01:47:05,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36769-0x10118200be10003 connected 2024-11-08T01:47:05,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-08T01:47:05,466 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-08T01:47:05,467 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T01:47:05,468 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.cache 2024-11-08T01:47:05,468 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T01:47:05,468 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T01:47:05,468 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T01:47:05,469 INFO [Thread-208 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T01:47:05,469 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T01:47:05,470 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T01:47:05,470 INFO [Thread-208 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-08T01:47:05,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T01:47:05,471 INFO [Thread-208 {}] bucket.BucketCache(1706): Retrieved 0 of chunks with blockCount = 0. 2024-11-08T01:47:05,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T01:47:05,475 DEBUG [Thread-208 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-08T01:47:05,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36769 2024-11-08T01:47:05,476 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T01:47:05,476 INFO [Thread-208 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 0 2024-11-08T01:47:05,476 INFO [Thread-208 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence is complete. 2024-11-08T01:47:05,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36769 2024-11-08T01:47:05,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36769 2024-11-08T01:47:05,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36769 2024-11-08T01:47:05,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36769 2024-11-08T01:47:05,478 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(746): ClusterId : 406e57db-4cc4-49a0-911e-1dc2177b2b5a 2024-11-08T01:47:05,478 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T01:47:05,560 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T01:47:05,560 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T01:47:05,587 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T01:47:05,587 DEBUG [RS:2;8118b061437f:36769 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59474859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8118b061437f/172.17.0.2:0 2024-11-08T01:47:05,601 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T01:47:05,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55723, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T01:47:05,604 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T01:47:05,604 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:47:05,605 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T01:47:05,605 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;8118b061437f:36769 2024-11-08T01:47:05,605 INFO [RS:2;8118b061437f:36769 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T01:47:05,605 INFO [RS:2;8118b061437f:36769 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T01:47:05,605 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T01:47:05,605 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C43839%2C1731030421786.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43839,1731030421786, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:47:05,606 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(2659): reportForDuty to master=8118b061437f,44413,1731030421572 with port=36769, startcode=1731030425449 2024-11-08T01:47:05,606 DEBUG [RS:2;8118b061437f:36769 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T01:47:05,608 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34341, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T01:47:05,608 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44413 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8118b061437f,36769,1731030425449 2024-11-08T01:47:05,609 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44413 {}] master.ServerManager(517): Registering regionserver=8118b061437f,36769,1731030425449 2024-11-08T01:47:05,610 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7 2024-11-08T01:47:05,610 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T01:47:05,610 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T01:47:05,622 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,43839,1731030421786/8118b061437f%2C43839%2C1731030421786.meta.1731030425606.meta 2024-11-08T01:47:05,622 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:47:05,622 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:47:05,622 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T01:47:05,622 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T01:47:05,623 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T01:47:05,623 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T01:47:05,623 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:05,623 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T01:47:05,623 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T01:47:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T01:47:05,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T01:47:05,624 DEBUG [RS:2;8118b061437f:36769 {}] zookeeper.ZKUtil(111): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8118b061437f,36769,1731030425449 2024-11-08T01:47:05,624 WARN [RS:2;8118b061437f:36769 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T01:47:05,624 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8118b061437f,36769,1731030425449] 2024-11-08T01:47:05,624 INFO [RS:2;8118b061437f:36769 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T01:47:05,624 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,36769,1731030425449 2024-11-08T01:47:05,624 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T01:47:05,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:05,626 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/d5d727aa2d1f49c78ea390ea95b95e2a, delay=949 ms 2024-11-08T01:47:05,627 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/d5d727aa2d1f49c78ea390ea95b95e2a 2024-11-08T01:47:05,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:05,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T01:47:05,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T01:47:05,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:05,629 INFO [RS:2;8118b061437f:36769 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T01:47:05,630 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/f55700072e304049b05f4b122e5e6f26, delay=918 ms 2024-11-08T01:47:05,631 INFO [RS:2;8118b061437f:36769 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T01:47:05,632 INFO [RS:2;8118b061437f:36769 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T01:47:05,632 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,632 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T01:47:05,632 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/f55700072e304049b05f4b122e5e6f26 2024-11-08T01:47:05,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:05,632 INFO [RS:2;8118b061437f:36769 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T01:47:05,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T01:47:05,632 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,632 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,632 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8118b061437f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T01:47:05,633 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8118b061437f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:47:05,633 DEBUG [RS:2;8118b061437f:36769 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8118b061437f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T01:47:05,634 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,634 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,634 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T01:47:05,634 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,634 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,634 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:05,634 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,634 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,36769,1731030425449-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T01:47:05,635 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/c11a1826e26f4f078cab664f5dfb1274, delay=995 ms 2024-11-08T01:47:05,636 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/c11a1826e26f4f078cab664f5dfb1274 2024-11-08T01:47:05,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T01:47:05,636 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T01:47:05,637 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:47:05,637 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740 2024-11-08T01:47:05,637 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T01:47:05,637 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T01:47:05,638 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T01:47:05,639 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T01:47:05,639 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74823545, jitterRate=0.11495770514011383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T01:47:05,639 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T01:47:05,640 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731030425623Writing region info on filesystem at 1731030425623Initializing all the Stores at 1731030425623Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030425623Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030425624 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030425624Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731030425624Cleaning up temporary data from old regions at 1731030425638 (+14 ms)Running coprocessor post-open hooks at 1731030425639 (+1 ms)Region opened successfully at 1731030425640 (+1 ms) 2024-11-08T01:47:05,641 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731030425601 2024-11-08T01:47:05,643 DEBUG [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T01:47:05,644 INFO [RS_OPEN_META-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T01:47:05,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=8118b061437f,43839,1731030421786 2024-11-08T01:47:05,644 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8118b061437f,43839,1731030421786, state=OPEN 2024-11-08T01:47:05,649 INFO [RS:2;8118b061437f:36769 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T01:47:05,649 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=8118b061437f,36769,1731030425449-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,649 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,650 INFO [RS:2;8118b061437f:36769 {}] regionserver.Replication(171): 8118b061437f,36769,1731030425449 started 2024-11-08T01:47:05,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:05,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:05,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T01:47:05,655 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8118b061437f,43839,1731030421786 2024-11-08T01:47:05,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:05,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:05,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T01:47:05,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-08T01:47:05,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8118b061437f,43839,1731030421786 in 207 msec 2024-11-08T01:47:05,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T01:47:05,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 373 msec 2024-11-08T01:47:05,661 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 8118b061437f,44791,1731030421752 had 2 regions 2024-11-08T01:47:05,661 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 8118b061437f,44791,1731030421752, splitWal=true, meta=true, isMeta: false 2024-11-08T01:47:05,662 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752-splitting dir is empty, no logs to split. 2024-11-08T01:47:05,662 INFO [PEWorker-3 {}] master.SplitWALManager(105): 8118b061437f,44791,1731030421752 WAL count=0, meta=false 2024-11-08T01:47:05,663 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,44791,1731030421752-splitting dir is empty, no logs to split. 2024-11-08T01:47:05,663 INFO [PEWorker-3 {}] master.SplitWALManager(105): 8118b061437f,44791,1731030421752 WAL count=0, meta=false 2024-11-08T01:47:05,663 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 8118b061437f,44791,1731030421752 WAL splitting is done? wals=0, meta=false 2024-11-08T01:47:05,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN}] 2024-11-08T01:47:05,665 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN 2024-11-08T01:47:05,666 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T01:47:05,666 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T01:47:05,666 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1482): Serving as 8118b061437f,36769,1731030425449, RpcServer on 8118b061437f/172.17.0.2:36769, sessionid=0x10118200be10003 2024-11-08T01:47:05,666 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T01:47:05,666 DEBUG [RS:2;8118b061437f:36769 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8118b061437f,36769,1731030425449 2024-11-08T01:47:05,667 DEBUG [RS:2;8118b061437f:36769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,36769,1731030425449' 2024-11-08T01:47:05,667 DEBUG [RS:2;8118b061437f:36769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T01:47:05,667 DEBUG [RS:2;8118b061437f:36769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T01:47:05,668 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T01:47:05,668 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T01:47:05,668 DEBUG [RS:2;8118b061437f:36769 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8118b061437f,36769,1731030425449 2024-11-08T01:47:05,668 DEBUG [RS:2;8118b061437f:36769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8118b061437f,36769,1731030425449' 2024-11-08T01:47:05,668 DEBUG [RS:2;8118b061437f:36769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T01:47:05,668 DEBUG [RS:2;8118b061437f:36769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T01:47:05,669 DEBUG [RS:2;8118b061437f:36769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T01:47:05,669 INFO [RS:2;8118b061437f:36769 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T01:47:05,669 INFO [RS:2;8118b061437f:36769 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T01:47:05,770 INFO [RS:2;8118b061437f:36769 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T01:47:05,772 INFO [RS:2;8118b061437f:36769 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8118b061437f%2C36769%2C1731030425449, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,36769,1731030425449, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs, maxLogs=55 2024-11-08T01:47:05,792 INFO [RS:2;8118b061437f:36769 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/WALs/8118b061437f,36769,1731030425449/8118b061437f%2C36769%2C1731030425449.1731030425773 2024-11-08T01:47:05,793 DEBUG [RS:2;8118b061437f:36769 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T01:47:05,816 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(204): Hosts are {8118b061437f=0} racks are {/default-rack=0} 2024-11-08T01:47:05,816 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T01:47:05,816 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T01:47:05,816 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T01:47:05,816 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T01:47:05,816 INFO [8118b061437f:44413 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T01:47:05,816 INFO [8118b061437f:44413 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T01:47:05,816 DEBUG [8118b061437f:44413 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T01:47:05,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4598fca782bf34f9926ed286f64ea7bf, regionState=OPENING, regionLocation=8118b061437f,36769,1731030425449 2024-11-08T01:47:05,818 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 8118b061437f:44791 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8118b061437f/172.17.0.2:44791 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T01:47:05,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8118b061437f,44791,1731030421752, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8118b061437f,44791,1731030421752, seqNum=-1, error=java.net.ConnectException: Call to address=8118b061437f:44791 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8118b061437f/172.17.0.2:44791 2024-11-08T01:47:05,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8118b061437f,44791,1731030421752, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T01:47:05,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8118b061437f,44791,1731030421752, seqNum=-1 from cache 2024-11-08T01:47:05,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 8118b061437f:44791 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8118b061437f/172.17.0.2:44791 2024-11-08T01:47:05,927 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T01:47:05,927 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8118b061437f,43839,1731030421786, seqNum=-1] 2024-11-08T01:47:05,927 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T01:47:05,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51491, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T01:47:05,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN because future has completed 2024-11-08T01:47:05,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4598fca782bf34f9926ed286f64ea7bf, server=8118b061437f,36769,1731030425449}] 2024-11-08T01:47:06,087 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T01:47:06,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T01:47:06,096 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,096 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 4598fca782bf34f9926ed286f64ea7bf, NAME => 'testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.', STARTKEY => '', ENDKEY => ''} 2024-11-08T01:47:06,097 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731030423239 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,098 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T01:47:06,098 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,098 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,099 INFO [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,100 INFO [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4598fca782bf34f9926ed286f64ea7bf columnFamilyName family 2024-11-08T01:47:06,100 DEBUG [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T01:47:06,102 DEBUG [StoreFileOpener-4598fca782bf34f9926ed286f64ea7bf-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e, delay=920 ms 2024-11-08T01:47:06,104 DEBUG [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e 2024-11-08T01:47:06,104 INFO [StoreOpener-4598fca782bf34f9926ed286f64ea7bf-1 {}] regionserver.HStore(327): Store=4598fca782bf34f9926ed286f64ea7bf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T01:47:06,104 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,106 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,107 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,107 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,107 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,109 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,109 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 4598fca782bf34f9926ed286f64ea7bf; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69712517, jitterRate=0.03879745304584503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T01:47:06,109 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,110 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 4598fca782bf34f9926ed286f64ea7bf: Running coprocessor pre-open hook at 1731030426098Writing region info on filesystem at 1731030426098Initializing all the Stores at 1731030426099 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731030426099Cleaning up temporary data from old regions at 1731030426107 (+8 ms)Running coprocessor post-open hooks at 1731030426109 (+2 ms)Region opened successfully at 1731030426110 (+1 ms) 2024-11-08T01:47:06,111 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf., pid=11, masterSystemTime=1731030426087 2024-11-08T01:47:06,113 DEBUG [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,113 INFO [RS_OPEN_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,114 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4598fca782bf34f9926ed286f64ea7bf, regionState=OPEN, openSeqNum=10, regionLocation=8118b061437f,36769,1731030425449 2024-11-08T01:47:06,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4598fca782bf34f9926ed286f64ea7bf, server=8118b061437f,36769,1731030425449 because future has completed 2024-11-08T01:47:06,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T01:47:06,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 4598fca782bf34f9926ed286f64ea7bf, server=8118b061437f,36769,1731030425449 in 182 msec 2024-11-08T01:47:06,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-08T01:47:06,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731030423239, region=4598fca782bf34f9926ed286f64ea7bf, ASSIGN in 454 msec 2024-11-08T01:47:06,120 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 8118b061437f,44791,1731030421752 after splitting done 2024-11-08T01:47:06,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 8118b061437f,44791,1731030421752, splitWal=true, meta=true in 849 msec 2024-11-08T01:47:06,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T01:47:06,167 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T01:47:06,167 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:06,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:06,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:06,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T01:47:06,168 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T01:47:06,168 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1764108974, stopped=false 2024-11-08T01:47:06,168 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8118b061437f,44413,1731030421572 2024-11-08T01:47:06,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:06,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:06,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T01:47:06,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:06,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:06,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:06,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T01:47:06,213 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T01:47:06,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:06,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:06,214 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:06,214 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8118b061437f,43839,1731030421786' ***** 2024-11-08T01:47:06,214 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T01:47:06,214 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8118b061437f,36769,1731030425449' ***** 2024-11-08T01:47:06,214 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T01:47:06,214 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:06,214 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T01:47:06,214 INFO [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T01:47:06,215 INFO [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T01:47:06,215 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T01:47:06,215 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T01:47:06,215 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T01:47:06,215 INFO [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence 2024-11-08T01:47:06,215 INFO [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T01:47:06,215 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T01:47:06,215 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T01:47:06,215 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T01:47:06,215 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T01:47:06,215 INFO [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/94d876f7-84e3-89a2-27b1-a5dcc6f32b03/bucket.persistence 2024-11-08T01:47:06,215 INFO [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T01:47:06,215 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T01:47:06,225 DEBUG [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1488): Thread RS:2;8118b061437f:36769 started persisting bucket cache to file 2024-11-08T01:47:06,225 DEBUG [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:06,225 DEBUG [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:06,225 DEBUG [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1488): Thread RS:1;8118b061437f:43839 started persisting bucket cache to file 2024-11-08T01:47:06,226 DEBUG [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T01:47:06,226 DEBUG [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T01:47:06,228 DEBUG [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T01:47:06,228 DEBUG [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T01:47:06,229 DEBUG [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731030426225 2024-11-08T01:47:06,229 DEBUG [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731030426225 2024-11-08T01:47:06,229 DEBUG [RS:1;8118b061437f:43839 {}] bucket.BucketCache(1510): Thread RS:1;8118b061437f:43839 finished persisting bucket cache to file, renaming 2024-11-08T01:47:06,229 INFO [RS:1;8118b061437f:43839 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T01:47:06,229 INFO [RS:1;8118b061437f:43839 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T01:47:06,229 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T01:47:06,229 INFO [RS:1;8118b061437f:43839 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T01:47:06,229 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(959): stopping server 8118b061437f,43839,1731030421786 2024-11-08T01:47:06,229 INFO [RS:1;8118b061437f:43839 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T01:47:06,229 INFO [RS:1;8118b061437f:43839 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;8118b061437f:43839. 2024-11-08T01:47:06,229 DEBUG [RS:1;8118b061437f:43839 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:06,229 DEBUG [RS:1;8118b061437f:43839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:06,230 INFO [RS:1;8118b061437f:43839 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T01:47:06,230 INFO [RS:1;8118b061437f:43839 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T01:47:06,230 INFO [RS:1;8118b061437f:43839 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T01:47:06,230 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T01:47:06,230 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T01:47:06,230 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T01:47:06,230 DEBUG [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T01:47:06,230 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T01:47:06,230 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T01:47:06,230 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T01:47:06,231 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T01:47:06,231 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T01:47:06,231 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T01:47:06,231 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-08T01:47:06,234 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T01:47:06,235 INFO [regionserver/8118b061437f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:06,236 INFO [regionserver/8118b061437f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:06,241 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c13318c432bb451a87986403664ead09 2024-11-08T01:47:06,246 DEBUG [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1510): Thread RS:2;8118b061437f:36769 finished persisting bucket cache to file, renaming 2024-11-08T01:47:06,246 WARN [RS:2;8118b061437f:36769 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-08T01:47:06,246 INFO [RS:2;8118b061437f:36769 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T01:47:06,246 INFO [RS:2;8118b061437f:36769 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T01:47:06,247 INFO [RS:2;8118b061437f:36769 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T01:47:06,247 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(3091): Received CLOSE for 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,247 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(959): stopping server 8118b061437f,36769,1731030425449 2024-11-08T01:47:06,247 INFO [RS:2;8118b061437f:36769 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T01:47:06,247 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T01:47:06,247 INFO [RS:2;8118b061437f:36769 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;8118b061437f:36769. 2024-11-08T01:47:06,247 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4598fca782bf34f9926ed286f64ea7bf, disabling compactions & flushes 2024-11-08T01:47:06,247 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,247 DEBUG [RS:2;8118b061437f:36769 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T01:47:06,247 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,248 DEBUG [RS:2;8118b061437f:36769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:06,248 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. after waiting 0 ms 2024-11-08T01:47:06,248 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,248 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T01:47:06,248 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1325): Online Regions={4598fca782bf34f9926ed286f64ea7bf=testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf.} 2024-11-08T01:47:06,248 DEBUG [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1351): Waiting on 4598fca782bf34f9926ed286f64ea7bf 2024-11-08T01:47:06,248 DEBUG [StoreFileCloser-4598fca782bf34f9926ed286f64ea7bf-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/family/07f0e4db5797446aa2d005de4929848e 2024-11-08T01:47:06,253 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/default/testBlockEvictionOnGracefulStop1731030423239/4598fca782bf34f9926ed286f64ea7bf/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T01:47:06,253 INFO [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,253 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4598fca782bf34f9926ed286f64ea7bf: Waiting for close lock at 1731030426247Running coprocessor pre-close hooks at 1731030426247Disabling compacts and flushes for region at 1731030426247Disabling writes for close at 1731030426248 (+1 ms)Writing region close event to WAL at 1731030426249 (+1 ms)Running coprocessor post-close hooks at 1731030426253 (+4 ms)Closed at 1731030426253 2024-11-08T01:47:06,254 DEBUG [RS_CLOSE_REGION-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf. 2024-11-08T01:47:06,256 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/c13318c432bb451a87986403664ead09 is 209, key is testBlockEvictionOnGracefulStop1731030423239,,1731030423244.4598fca782bf34f9926ed286f64ea7bf./info:regioninfo/1731030426113/Put/seqid=0 2024-11-08T01:47:06,257 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/c13318c432bb451a87986403664ead09 2024-11-08T01:47:06,259 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/.tmp/info/c13318c432bb451a87986403664ead09 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/c13318c432bb451a87986403664ead09 2024-11-08T01:47:06,260 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/c13318c432bb451a87986403664ead09, delay=958 ms 2024-11-08T01:47:06,261 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/c13318c432bb451a87986403664ead09, entries=8, sequenceid=19, filesize=6.6 K 2024-11-08T01:47:06,261 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false 2024-11-08T01:47:06,262 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/d5d727aa2d1f49c78ea390ea95b95e2a 2024-11-08T01:47:06,262 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/info/c13318c432bb451a87986403664ead09 2024-11-08T01:47:06,263 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/ns/f55700072e304049b05f4b122e5e6f26 2024-11-08T01:47:06,264 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/table/c11a1826e26f4f078cab664f5dfb1274 2024-11-08T01:47:06,268 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-08T01:47:06,268 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T01:47:06,269 INFO [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T01:47:06,269 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731030426230Running coprocessor pre-close hooks at 1731030426230Disabling compacts and flushes for region at 1731030426230Disabling writes for close at 1731030426231 (+1 ms)Obtaining lock to block concurrent updates at 1731030426231Preparing flush snapshotting stores in 1588230740 at 1731030426231Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731030426231Flushing stores of hbase:meta,,1.1588230740 at 1731030426232 (+1 ms)Flushing 1588230740/info: creating writer at 1731030426232Flushing 1588230740/info: appending metadata at 1731030426256 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731030426256Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12030077: reopening flushed file at 1731030426259 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false at 1731030426261 (+2 ms)Writing region close event to WAL at 1731030426264 (+3 ms)Running coprocessor post-close hooks at 1731030426268 (+4 ms)Closed at 1731030426268 2024-11-08T01:47:06,269 DEBUG [RS_CLOSE_META-regionserver/8118b061437f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T01:47:06,431 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(976): stopping server 8118b061437f,43839,1731030421786; all regions closed. 2024-11-08T01:47:06,437 DEBUG [RS:1;8118b061437f:43839 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:06,437 INFO [RS:1;8118b061437f:43839 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C43839%2C1731030421786.meta:.meta(num 1731030425606) 2024-11-08T01:47:06,441 DEBUG [RS:1;8118b061437f:43839 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:06,441 INFO [RS:1;8118b061437f:43839 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C43839%2C1731030421786:(num 1731030422868) 2024-11-08T01:47:06,441 DEBUG [RS:1;8118b061437f:43839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:06,441 INFO [RS:1;8118b061437f:43839 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:06,441 INFO [RS:1;8118b061437f:43839 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T01:47:06,442 INFO [RS:1;8118b061437f:43839 {}] hbase.ChoreService(370): Chore service for: regionserver/8118b061437f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T01:47:06,442 INFO [RS:1;8118b061437f:43839 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T01:47:06,442 INFO [regionserver/8118b061437f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T01:47:06,442 INFO [RS:1;8118b061437f:43839 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43839 2024-11-08T01:47:06,448 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(976): stopping server 8118b061437f,36769,1731030425449; all regions closed. 2024-11-08T01:47:06,450 DEBUG [RS:2;8118b061437f:36769 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/oldWALs 2024-11-08T01:47:06,450 INFO [RS:2;8118b061437f:36769 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8118b061437f%2C36769%2C1731030425449:(num 1731030425773) 2024-11-08T01:47:06,450 DEBUG [RS:2;8118b061437f:36769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T01:47:06,450 INFO [RS:2;8118b061437f:36769 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T01:47:06,450 INFO [RS:2;8118b061437f:36769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T01:47:06,450 INFO [RS:2;8118b061437f:36769 {}] hbase.ChoreService(370): Chore service for: regionserver/8118b061437f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T01:47:06,451 INFO [RS:2;8118b061437f:36769 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T01:47:06,451 INFO [RS:2;8118b061437f:36769 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T01:47:06,451 INFO [regionserver/8118b061437f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T01:47:06,451 INFO [RS:2;8118b061437f:36769 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T01:47:06,451 INFO [RS:2;8118b061437f:36769 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T01:47:06,451 INFO [RS:2;8118b061437f:36769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36769 2024-11-08T01:47:06,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T01:47:06,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8118b061437f,43839,1731030421786 2024-11-08T01:47:06,459 INFO [RS:1;8118b061437f:43839 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T01:47:06,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8118b061437f,36769,1731030425449 2024-11-08T01:47:06,469 INFO [RS:2;8118b061437f:36769 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T01:47:06,480 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8118b061437f,43839,1731030421786] 2024-11-08T01:47:06,501 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8118b061437f,43839,1731030421786 already deleted, retry=false 2024-11-08T01:47:06,501 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8118b061437f,43839,1731030421786 expired; onlineServers=1 2024-11-08T01:47:06,501 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8118b061437f,36769,1731030425449] 2024-11-08T01:47:06,511 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8118b061437f,36769,1731030425449 already deleted, retry=false 2024-11-08T01:47:06,511 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8118b061437f,36769,1731030425449 expired; onlineServers=0 2024-11-08T01:47:06,511 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8118b061437f,44413,1731030421572' ***** 2024-11-08T01:47:06,511 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T01:47:06,511 INFO [M:0;8118b061437f:44413 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T01:47:06,511 INFO [M:0;8118b061437f:44413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T01:47:06,512 DEBUG [M:0;8118b061437f:44413 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T01:47:06,512 DEBUG [M:0;8118b061437f:44413 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T01:47:06,512 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T01:47:06,512 DEBUG [master/8118b061437f:0:becomeActiveMaster-HFileCleaner.small.0-1731030422453 {}] cleaner.HFileCleaner(306): Exit Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.small.0-1731030422453,5,FailOnTimeoutGroup] 2024-11-08T01:47:06,512 DEBUG [master/8118b061437f:0:becomeActiveMaster-HFileCleaner.large.0-1731030422452 {}] cleaner.HFileCleaner(306): Exit Thread[master/8118b061437f:0:becomeActiveMaster-HFileCleaner.large.0-1731030422452,5,FailOnTimeoutGroup] 2024-11-08T01:47:06,512 INFO [M:0;8118b061437f:44413 {}] hbase.ChoreService(370): Chore service for: master/8118b061437f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T01:47:06,512 INFO [M:0;8118b061437f:44413 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T01:47:06,512 DEBUG [M:0;8118b061437f:44413 {}] master.HMaster(1795): Stopping service threads 2024-11-08T01:47:06,512 INFO [M:0;8118b061437f:44413 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T01:47:06,512 INFO [M:0;8118b061437f:44413 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T01:47:06,512 INFO [M:0;8118b061437f:44413 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T01:47:06,512 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T01:47:06,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T01:47:06,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T01:47:06,522 DEBUG [M:0;8118b061437f:44413 {}] zookeeper.ZKUtil(347): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T01:47:06,522 WARN [M:0;8118b061437f:44413 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T01:47:06,522 INFO [M:0;8118b061437f:44413 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/.lastflushedseqids 2024-11-08T01:47:06,525 INFO [M:0;8118b061437f:44413 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T01:47:06,525 INFO [M:0;8118b061437f:44413 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T01:47:06,525 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T01:47:06,525 INFO [M:0;8118b061437f:44413 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:06,525 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:06,525 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T01:47:06,525 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:06,526 INFO [M:0;8118b061437f:44413 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.49 KB heapSize=49.88 KB 2024-11-08T01:47:06,534 DEBUG [M:0;8118b061437f:44413 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6eec3f4dc4a54067b56c913cfd183806 2024-11-08T01:47:06,546 DEBUG [M:0;8118b061437f:44413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6eec3f4dc4a54067b56c913cfd183806 is 82, key is hbase:meta,,1/info:regioninfo/1731030425644/Put/seqid=0 2024-11-08T01:47:06,547 INFO [M:0;8118b061437f:44413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6eec3f4dc4a54067b56c913cfd183806 2024-11-08T01:47:06,555 DEBUG [M:0;8118b061437f:44413 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f1d8be12c3ba41daa15a79324ca89aee 2024-11-08T01:47:06,570 DEBUG [M:0;8118b061437f:44413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1d8be12c3ba41daa15a79324ca89aee is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731030423627/Put/seqid=0 2024-11-08T01:47:06,570 INFO [M:0;8118b061437f:44413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.26 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1d8be12c3ba41daa15a79324ca89aee 2024-11-08T01:47:06,578 DEBUG [M:0;8118b061437f:44413 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f347b80a644458aa4f194dcb825df00 2024-11-08T01:47:06,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:06,580 INFO [RS:1;8118b061437f:43839 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T01:47:06,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43839-0x10118200be10002, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:06,580 INFO [RS:1;8118b061437f:43839 {}] regionserver.HRegionServer(1031): Exiting; stopping=8118b061437f,43839,1731030421786; zookeeper connection closed. 2024-11-08T01:47:06,580 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e12ec59 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e12ec59 2024-11-08T01:47:06,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:06,590 INFO [RS:2;8118b061437f:36769 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T01:47:06,591 INFO [RS:2;8118b061437f:36769 {}] regionserver.HRegionServer(1031): Exiting; stopping=8118b061437f,36769,1731030425449; zookeeper connection closed. 2024-11-08T01:47:06,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36769-0x10118200be10003, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:06,591 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5043eb61 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5043eb61 2024-11-08T01:47:06,591 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T01:47:06,599 DEBUG [M:0;8118b061437f:44413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f347b80a644458aa4f194dcb825df00 is 69, key is 8118b061437f,36769,1731030425449/rs:state/1731030425609/Put/seqid=0 2024-11-08T01:47:06,600 INFO [M:0;8118b061437f:44413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f347b80a644458aa4f194dcb825df00 2024-11-08T01:47:06,602 INFO [M:0;8118b061437f:44413 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1f347b80a644458aa4f194dcb825df00 2024-11-08T01:47:06,603 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6eec3f4dc4a54067b56c913cfd183806 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6eec3f4dc4a54067b56c913cfd183806 2024-11-08T01:47:06,605 INFO [M:0;8118b061437f:44413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6eec3f4dc4a54067b56c913cfd183806, entries=14, sequenceid=102, filesize=5.9 K 2024-11-08T01:47:06,606 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1d8be12c3ba41daa15a79324ca89aee as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1d8be12c3ba41daa15a79324ca89aee 2024-11-08T01:47:06,608 INFO [M:0;8118b061437f:44413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1d8be12c3ba41daa15a79324ca89aee, entries=11, sequenceid=102, filesize=6.5 K 2024-11-08T01:47:06,609 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f347b80a644458aa4f194dcb825df00 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1f347b80a644458aa4f194dcb825df00 2024-11-08T01:47:06,611 INFO [M:0;8118b061437f:44413 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1f347b80a644458aa4f194dcb825df00 2024-11-08T01:47:06,612 INFO [M:0;8118b061437f:44413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d62cd30e-0d85-42e1-1b58-4986f73b2c6c/2f647dc1-c54d-8372-32d7-d67709f30df7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1f347b80a644458aa4f194dcb825df00, entries=3, sequenceid=102, filesize=5.3 K 2024-11-08T01:47:06,613 INFO [M:0;8118b061437f:44413 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=102, compaction requested=false 2024-11-08T01:47:06,617 INFO [M:0;8118b061437f:44413 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T01:47:06,617 DEBUG [M:0;8118b061437f:44413 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731030426525Disabling compacts and flushes for region at 1731030426525Disabling writes for close at 1731030426525Obtaining lock to block concurrent updates at 1731030426526 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731030426526Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40436, getHeapSize=51016, getOffHeapSize=0, getCellsCount=127 at 1731030426526Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731030426526Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731030426526Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731030426546 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731030426546Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731030426549 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731030426569 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731030426569Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731030426572 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731030426598 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731030426598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bcc7467: reopening flushed file at 1731030426603 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e895a03: reopening flushed file at 1731030426606 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16814e1e: reopening flushed file at 1731030426609 (+3 ms)Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=102, compaction requested=false at 1731030426613 (+4 ms)Writing region close event to WAL at 1731030426617 (+4 ms)Closed at 1731030426617 2024-11-08T01:47:06,622 INFO [M:0;8118b061437f:44413 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T01:47:06,623 INFO [M:0;8118b061437f:44413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44413 2024-11-08T01:47:06,623 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T01:47:06,624 INFO [M:0;8118b061437f:44413 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T01:47:06,735 INFO [M:0;8118b061437f:44413 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T01:47:06,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:06,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44413-0x10118200be10000, quorum=127.0.0.1:49616, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T01:47:06,737 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T01:47:06,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T01:47:06,739 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T01:47:06,746 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=414 (was 450), ProcessCount=11 (was 11), AvailableMemoryMB=7819 (was 7910)