2024-11-10 12:28:00,497 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-10 12:28:00,508 main DEBUG Took 0.009793 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 12:28:00,509 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 12:28:00,509 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 12:28:00,510 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 12:28:00,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,518 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 12:28:00,540 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,541 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,542 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,543 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,543 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,545 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,546 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,548 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,549 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,550 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,551 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,552 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,553 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,554 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 12:28:00,555 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,556 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 12:28:00,558 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 12:28:00,560 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 12:28:00,561 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 12:28:00,562 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 12:28:00,563 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 12:28:00,563 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 12:28:00,571 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 12:28:00,574 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 12:28:00,575 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 12:28:00,575 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 12:28:00,576 main DEBUG createAppenders(={Console}) 2024-11-10 12:28:00,576 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-10 12:28:00,577 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-10 12:28:00,577 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-10 12:28:00,577 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 12:28:00,578 main DEBUG OutputStream closed 2024-11-10 12:28:00,578 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 12:28:00,578 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 12:28:00,578 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-10 12:28:00,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 12:28:00,654 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 12:28:00,655 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 12:28:00,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 12:28:00,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 12:28:00,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 12:28:00,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 12:28:00,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 12:28:00,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 12:28:00,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 12:28:00,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 12:28:00,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 12:28:00,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 12:28:00,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 12:28:00,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 12:28:00,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 12:28:00,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 12:28:00,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 12:28:00,665 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 12:28:00,666 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-10 12:28:00,666 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 12:28:00,667 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-10T12:28:00,686 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-10 12:28:00,690 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 12:28:00,690 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T12:28:00,985 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669 2024-11-10T12:28:01,007 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=286, ProcessCount=11, AvailableMemoryMB=9317 2024-11-10T12:28:01,102 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T12:28:01,190 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:01,207 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/cluster_91c56e46-31ff-85b6-a73a-0cbdce45e1aa, deleteOnExit=true 2024-11-10T12:28:01,312 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/cluster_91c56e46-31ff-85b6-a73a-0cbdce45e1aa/zookeeper_0, clientPort=50874, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/cluster_91c56e46-31ff-85b6-a73a-0cbdce45e1aa/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/cluster_91c56e46-31ff-85b6-a73a-0cbdce45e1aa/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T12:28:01,325 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50874 2024-11-10T12:28:01,326 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:01,414 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9 with version=8 2024-11-10T12:28:01,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/hbase-staging 2024-11-10T12:28:01,515 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T12:28:01,799 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3857ccc89b65:0 server-side Connection retries=45 2024-11-10T12:28:01,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T12:28:01,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T12:28:01,820 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T12:28:01,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T12:28:01,821 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T12:28:01,959 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T12:28:02,050 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T12:28:02,090 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T12:28:02,101 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T12:28:02,105 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T12:28:02,147 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28769 (auto-detected) 2024-11-10T12:28:02,149 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T12:28:02,177 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43729 2024-11-10T12:28:02,200 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43729 connecting to ZooKeeper ensemble=127.0.0.1:50874 2024-11-10T12:28:02,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:437290x0, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T12:28:02,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43729-0x10101d98c100000 connected 2024-11-10T12:28:02,269 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:02,270 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:02,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T12:28:02,283 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9, hbase.cluster.distributed=false 2024-11-10T12:28:02,307 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T12:28:02,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43729 2024-11-10T12:28:02,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43729 2024-11-10T12:28:02,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43729 2024-11-10T12:28:02,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43729 2024-11-10T12:28:02,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43729 2024-11-10T12:28:02,405 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3857ccc89b65:0 server-side Connection retries=45 2024-11-10T12:28:02,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T12:28:02,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T12:28:02,408 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T12:28:02,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T12:28:02,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T12:28:02,411 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T12:28:02,414 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T12:28:02,416 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45393 2024-11-10T12:28:02,418 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45393 connecting to ZooKeeper ensemble=127.0.0.1:50874 2024-11-10T12:28:02,418 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:02,419 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:02,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453930x0, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T12:28:02,425 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45393-0x10101d98c100001 connected 2024-11-10T12:28:02,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T12:28:02,431 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T12:28:02,444 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/bucket.cache 2024-11-10T12:28:02,449 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T12:28:02,450 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T12:28:02,452 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-10T12:28:02,454 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T12:28:02,454 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T12:28:02,459 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T12:28:02,465 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T12:28:02,466 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T12:28:02,466 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/bucket.persistence is complete. 2024-11-10T12:28:02,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T12:28:02,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T12:28:02,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-10T12:28:02,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45393 2024-11-10T12:28:02,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45393 2024-11-10T12:28:02,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-10T12:28:02,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-10T12:28:02,509 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3857ccc89b65:43729 2024-11-10T12:28:02,513 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3857ccc89b65,43729,1731241681587 2024-11-10T12:28:02,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T12:28:02,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T12:28:02,524 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3857ccc89b65,43729,1731241681587 2024-11-10T12:28:02,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T12:28:02,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:02,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:02,551 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T12:28:02,552 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3857ccc89b65,43729,1731241681587 from backup master directory 2024-11-10T12:28:02,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3857ccc89b65,43729,1731241681587 2024-11-10T12:28:02,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T12:28:02,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T12:28:02,556 WARN [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T12:28:02,557 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3857ccc89b65,43729,1731241681587 2024-11-10T12:28:02,560 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T12:28:02,562 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T12:28:02,572 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/hbase.id] with ID: 26bdd638-5c70-492a-9b2e-d344f6accf73 2024-11-10T12:28:02,572 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/.tmp/hbase.id 2024-11-10T12:28:02,579 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/hbase.id] 2024-11-10T12:28:02,631 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:02,637 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T12:28:02,657 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-10T12:28:02,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:02,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:02,691 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T12:28:02,694 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T12:28:02,743 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T12:28:02,749 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T12:28:02,808 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store 2024-11-10T12:28:02,830 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T12:28:02,833 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T12:28:02,835 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T12:28:02,835 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:02,835 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:02,836 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T12:28:02,837 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:02,837 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:02,837 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731241682834Disabling compacts and flushes for region at 1731241682834Disabling writes for close at 1731241682837 (+3 ms)Writing region close event to WAL at 1731241682837Closed at 1731241682837 2024-11-10T12:28:02,838 WARN [master/3857ccc89b65:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/.initializing 2024-11-10T12:28:02,838 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/WALs/3857ccc89b65,43729,1731241681587 2024-11-10T12:28:02,852 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T12:28:02,868 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3857ccc89b65%2C43729%2C1731241681587, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/WALs/3857ccc89b65,43729,1731241681587, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/oldWALs, maxLogs=10 2024-11-10T12:28:02,911 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/WALs/3857ccc89b65,43729,1731241681587/3857ccc89b65%2C43729%2C1731241681587.1731241682873 2024-11-10T12:28:02,912 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T12:28:02,913 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T12:28:02,914 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T12:28:02,918 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:02,919 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:02,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T12:28:03,008 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:03,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T12:28:03,018 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T12:28:03,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T12:28:03,026 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T12:28:03,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,032 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T12:28:03,032 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T12:28:03,033 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,037 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,038 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,041 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,042 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,045 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T12:28:03,047 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T12:28:03,055 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T12:28:03,055 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69292167, jitterRate=0.03253374993801117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T12:28:03,062 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731241682933Initializing all the Stores at 1731241682933Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731241682934 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731241682935 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731241682935Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731241682935Cleaning up temporary data from old regions at 1731241683042 (+107 ms)Region opened successfully at 1731241683062 (+20 ms) 2024-11-10T12:28:03,064 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T12:28:03,101 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a30fe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3857ccc89b65/172.17.0.2:0 2024-11-10T12:28:03,142 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T12:28:03,155 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T12:28:03,155 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T12:28:03,158 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T12:28:03,159 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T12:28:03,166 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-10T12:28:03,166 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T12:28:03,192 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T12:28:03,202 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T12:28:03,204 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T12:28:03,207 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T12:28:03,209 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T12:28:03,211 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T12:28:03,214 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T12:28:03,218 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T12:28:03,220 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T12:28:03,222 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T12:28:03,225 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T12:28:03,246 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T12:28:03,248 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T12:28:03,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T12:28:03,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T12:28:03,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,265 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3857ccc89b65,43729,1731241681587, sessionid=0x10101d98c100000, setting cluster-up flag (Was=false) 2024-11-10T12:28:03,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,289 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T12:28:03,291 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3857ccc89b65,43729,1731241681587 2024-11-10T12:28:03,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,303 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T12:28:03,305 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3857ccc89b65,43729,1731241681587 2024-11-10T12:28:03,310 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T12:28:03,388 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(746): ClusterId : 26bdd638-5c70-492a-9b2e-d344f6accf73 2024-11-10T12:28:03,390 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T12:28:03,396 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T12:28:03,396 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T12:28:03,400 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T12:28:03,400 DEBUG [RS:0;3857ccc89b65:45393 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736f9129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3857ccc89b65/172.17.0.2:0 2024-11-10T12:28:03,401 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T12:28:03,412 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T12:28:03,417 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3857ccc89b65:45393 2024-11-10T12:28:03,419 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T12:28:03,420 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T12:28:03,421 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T12:28:03,421 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T12:28:03,424 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(2659): reportForDuty to master=3857ccc89b65,43729,1731241681587 with port=45393, startcode=1731241682366 2024-11-10T12:28:03,426 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3857ccc89b65,43729,1731241681587 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T12:28:03,437 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3857ccc89b65:0, corePoolSize=5, maxPoolSize=5 2024-11-10T12:28:03,437 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3857ccc89b65:0, corePoolSize=5, maxPoolSize=5 2024-11-10T12:28:03,438 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3857ccc89b65:0, corePoolSize=5, maxPoolSize=5 2024-11-10T12:28:03,438 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3857ccc89b65:0, corePoolSize=5, maxPoolSize=5 2024-11-10T12:28:03,438 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3857ccc89b65:0, corePoolSize=10, maxPoolSize=10 2024-11-10T12:28:03,439 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,439 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3857ccc89b65:0, corePoolSize=2, maxPoolSize=2 2024-11-10T12:28:03,439 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,441 DEBUG [RS:0;3857ccc89b65:45393 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T12:28:03,445 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731241713445 2024-11-10T12:28:03,445 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T12:28:03,446 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T12:28:03,447 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T12:28:03,449 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T12:28:03,452 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,453 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T12:28:03,453 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T12:28:03,453 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T12:28:03,453 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T12:28:03,454 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T12:28:03,454 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,457 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T12:28:03,459 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T12:28:03,459 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T12:28:03,462 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T12:28:03,463 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T12:28:03,464 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3857ccc89b65:0:becomeActiveMaster-HFileCleaner.large.0-1731241683464,5,FailOnTimeoutGroup] 2024-11-10T12:28:03,467 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3857ccc89b65:0:becomeActiveMaster-HFileCleaner.small.0-1731241683465,5,FailOnTimeoutGroup] 2024-11-10T12:28:03,467 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,467 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T12:28:03,468 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,468 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,480 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T12:28:03,480 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9 2024-11-10T12:28:03,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T12:28:03,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T12:28:03,501 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T12:28:03,501 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:03,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T12:28:03,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T12:28:03,510 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:03,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T12:28:03,515 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T12:28:03,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:03,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T12:28:03,517 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T12:28:03,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T12:28:03,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:03,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:03,522 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T12:28:03,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740 2024-11-10T12:28:03,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740 2024-11-10T12:28:03,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T12:28:03,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T12:28:03,525 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43729 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3857ccc89b65,45393,1731241682366 2024-11-10T12:28:03,525 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T12:28:03,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T12:28:03,528 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43729 {}] master.ServerManager(517): Registering regionserver=3857ccc89b65,45393,1731241682366 2024-11-10T12:28:03,534 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T12:28:03,534 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73178726, jitterRate=0.09044799208641052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T12:28:03,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731241683491Initializing all the Stores at 1731241683491Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731241683492 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731241683494 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731241683494Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731241683494Cleaning up temporary data from old regions at 1731241683525 (+31 ms)Region opened successfully at 1731241683539 (+14 ms) 2024-11-10T12:28:03,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T12:28:03,540 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T12:28:03,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T12:28:03,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T12:28:03,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T12:28:03,542 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T12:28:03,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731241683539Disabling compacts and flushes for region at 1731241683539Disabling writes for close at 1731241683540 (+1 ms)Writing region close event to WAL at 1731241683541 (+1 ms)Closed at 1731241683542 (+1 ms) 2024-11-10T12:28:03,545 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T12:28:03,545 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T12:28:03,547 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9 2024-11-10T12:28:03,547 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T12:28:03,547 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T12:28:03,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T12:28:03,554 DEBUG [RS:0;3857ccc89b65:45393 {}] zookeeper.ZKUtil(111): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3857ccc89b65,45393,1731241682366 2024-11-10T12:28:03,554 WARN [RS:0;3857ccc89b65:45393 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T12:28:03,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T12:28:03,554 INFO [RS:0;3857ccc89b65:45393 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T12:28:03,554 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366 2024-11-10T12:28:03,556 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3857ccc89b65,45393,1731241682366] 2024-11-10T12:28:03,563 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T12:28:03,565 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T12:28:03,582 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T12:28:03,596 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T12:28:03,601 INFO [RS:0;3857ccc89b65:45393 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T12:28:03,601 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,602 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T12:28:03,607 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T12:28:03,608 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,609 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,609 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,609 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,609 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,609 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,609 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3857ccc89b65:0, corePoolSize=2, maxPoolSize=2 2024-11-10T12:28:03,610 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,610 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,610 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,610 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,610 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,610 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3857ccc89b65:0, corePoolSize=1, maxPoolSize=1 2024-11-10T12:28:03,610 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3857ccc89b65:0, corePoolSize=3, maxPoolSize=3 2024-11-10T12:28:03,611 DEBUG [RS:0;3857ccc89b65:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3857ccc89b65:0, corePoolSize=3, maxPoolSize=3 2024-11-10T12:28:03,612 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,612 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,612 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,612 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,612 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,612 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,45393,1731241682366-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T12:28:03,634 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T12:28:03,637 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,45393,1731241682366-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,637 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,637 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.Replication(171): 3857ccc89b65,45393,1731241682366 started 2024-11-10T12:28:03,659 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:03,660 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1482): Serving as 3857ccc89b65,45393,1731241682366, RpcServer on 3857ccc89b65/172.17.0.2:45393, sessionid=0x10101d98c100001 2024-11-10T12:28:03,661 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T12:28:03,661 DEBUG [RS:0;3857ccc89b65:45393 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3857ccc89b65,45393,1731241682366 2024-11-10T12:28:03,661 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3857ccc89b65,45393,1731241682366' 2024-11-10T12:28:03,661 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T12:28:03,663 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T12:28:03,664 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T12:28:03,664 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T12:28:03,664 DEBUG [RS:0;3857ccc89b65:45393 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3857ccc89b65,45393,1731241682366 2024-11-10T12:28:03,664 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3857ccc89b65,45393,1731241682366' 2024-11-10T12:28:03,664 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T12:28:03,665 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T12:28:03,666 DEBUG [RS:0;3857ccc89b65:45393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T12:28:03,666 INFO [RS:0;3857ccc89b65:45393 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T12:28:03,666 INFO [RS:0;3857ccc89b65:45393 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T12:28:03,717 WARN [3857ccc89b65:43729 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T12:28:03,772 INFO [RS:0;3857ccc89b65:45393 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T12:28:03,774 INFO [RS:0;3857ccc89b65:45393 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3857ccc89b65%2C45393%2C1731241682366, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/oldWALs, maxLogs=55 2024-11-10T12:28:03,799 INFO [RS:0;3857ccc89b65:45393 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366/3857ccc89b65%2C45393%2C1731241682366.1731241683777 2024-11-10T12:28:03,800 DEBUG [RS:0;3857ccc89b65:45393 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T12:28:03,969 DEBUG [3857ccc89b65:43729 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T12:28:03,983 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3857ccc89b65,45393,1731241682366 2024-11-10T12:28:03,988 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3857ccc89b65,45393,1731241682366, state=OPENING 2024-11-10T12:28:03,994 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T12:28:03,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:03,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T12:28:03,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T12:28:03,998 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T12:28:04,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3857ccc89b65,45393,1731241682366}] 2024-11-10T12:28:04,174 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T12:28:04,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43551, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T12:28:04,188 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T12:28:04,188 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T12:28:04,189 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T12:28:04,190 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3857ccc89b65%2C45393%2C1731241682366.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/oldWALs, maxLogs=55 2024-11-10T12:28:04,214 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366/3857ccc89b65%2C45393%2C1731241682366.meta.1731241684192.meta 2024-11-10T12:28:04,215 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T12:28:04,215 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T12:28:04,217 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T12:28:04,219 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T12:28:04,224 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T12:28:04,229 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T12:28:04,230 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T12:28:04,230 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T12:28:04,230 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T12:28:04,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T12:28:04,233 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T12:28:04,233 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:04,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:04,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T12:28:04,234 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T12:28:04,234 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:04,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:04,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T12:28:04,235 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T12:28:04,235 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:04,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:04,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T12:28:04,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T12:28:04,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:04,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T12:28:04,237 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T12:28:04,238 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740 2024-11-10T12:28:04,239 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740 2024-11-10T12:28:04,239 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T12:28:04,240 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T12:28:04,240 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T12:28:04,241 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T12:28:04,242 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63359725, jitterRate=-0.05586652457714081}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T12:28:04,242 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T12:28:04,243 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731241684230Writing region info on filesystem at 1731241684231 (+1 ms)Initializing all the Stores at 1731241684231Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731241684231Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731241684231Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731241684231Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731241684231Cleaning up temporary data from old regions at 1731241684240 (+9 ms)Running coprocessor post-open hooks at 1731241684242 (+2 ms)Region opened successfully at 1731241684243 (+1 ms) 2024-11-10T12:28:04,250 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731241684165 2024-11-10T12:28:04,260 DEBUG [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T12:28:04,261 INFO [RS_OPEN_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T12:28:04,262 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3857ccc89b65,45393,1731241682366 2024-11-10T12:28:04,263 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3857ccc89b65,45393,1731241682366, state=OPEN 2024-11-10T12:28:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T12:28:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T12:28:04,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T12:28:04,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T12:28:04,270 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3857ccc89b65,45393,1731241682366 2024-11-10T12:28:04,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T12:28:04,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3857ccc89b65,45393,1731241682366 in 270 msec 2024-11-10T12:28:04,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T12:28:04,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 721 msec 2024-11-10T12:28:04,278 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T12:28:04,278 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T12:28:04,299 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T12:28:04,300 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3857ccc89b65,45393,1731241682366, seqNum=-1] 2024-11-10T12:28:04,320 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T12:28:04,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53839, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T12:28:04,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 996 msec 2024-11-10T12:28:04,350 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731241684350, completionTime=-1 2024-11-10T12:28:04,353 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T12:28:04,353 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T12:28:04,378 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T12:28:04,378 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731241744378 2024-11-10T12:28:04,379 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731241804379 2024-11-10T12:28:04,379 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 25 msec 2024-11-10T12:28:04,381 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,43729,1731241681587-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:04,382 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,43729,1731241681587-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:04,382 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,43729,1731241681587-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:04,383 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3857ccc89b65:43729, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:04,384 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:04,384 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:04,389 DEBUG [master/3857ccc89b65:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T12:28:04,409 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.852sec 2024-11-10T12:28:04,410 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T12:28:04,411 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T12:28:04,412 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T12:28:04,413 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T12:28:04,413 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T12:28:04,414 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,43729,1731241681587-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T12:28:04,414 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,43729,1731241681587-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T12:28:04,419 DEBUG [master/3857ccc89b65:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T12:28:04,419 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T12:28:04,420 INFO [master/3857ccc89b65:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3857ccc89b65,43729,1731241681587-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T12:28:04,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567b1492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T12:28:04,500 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T12:28:04,500 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T12:28:04,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3857ccc89b65,43729,-1 for getting cluster id 2024-11-10T12:28:04,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T12:28:04,516 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26bdd638-5c70-492a-9b2e-d344f6accf73' 2024-11-10T12:28:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T12:28:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26bdd638-5c70-492a-9b2e-d344f6accf73" 2024-11-10T12:28:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca0d679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T12:28:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3857ccc89b65,43729,-1] 2024-11-10T12:28:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T12:28:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T12:28:04,526 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T12:28:04,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3d5ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T12:28:04,530 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T12:28:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3857ccc89b65,45393,1731241682366, seqNum=-1] 2024-11-10T12:28:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T12:28:04,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T12:28:04,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3857ccc89b65,43729,1731241681587 2024-11-10T12:28:04,568 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T12:28:04,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3857ccc89b65,43729,1731241681587 2024-11-10T12:28:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4216f89f 2024-11-10T12:28:04,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T12:28:04,583 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T12:28:04,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T12:28:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-10T12:28:04,597 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T12:28:04,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-10T12:28:04,599 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:04,600 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T12:28:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T12:28:04,637 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef177eb06ca366a257ef64b1dd9fbfa3, NAME => 'table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9 2024-11-10T12:28:04,645 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T12:28:04,645 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing ef177eb06ca366a257ef64b1dd9fbfa3, disabling compactions & flushes 2024-11-10T12:28:04,645 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:04,645 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:04,645 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. after waiting 0 ms 2024-11-10T12:28:04,645 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:04,645 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:04,645 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef177eb06ca366a257ef64b1dd9fbfa3: Waiting for close lock at 1731241684645Disabling compacts and flushes for region at 1731241684645Disabling writes for close at 1731241684645Writing region close event to WAL at 1731241684645Closed at 1731241684645 2024-11-10T12:28:04,647 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T12:28:04,652 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731241684647"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731241684647"}]},"ts":"1731241684647"} 2024-11-10T12:28:04,658 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T12:28:04,659 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T12:28:04,662 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731241684659"}]},"ts":"1731241684659"} 2024-11-10T12:28:04,667 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-10T12:28:04,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=ef177eb06ca366a257ef64b1dd9fbfa3, ASSIGN}] 2024-11-10T12:28:04,670 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=ef177eb06ca366a257ef64b1dd9fbfa3, ASSIGN 2024-11-10T12:28:04,671 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=ef177eb06ca366a257ef64b1dd9fbfa3, ASSIGN; state=OFFLINE, location=3857ccc89b65,45393,1731241682366; forceNewPlan=false, retain=false 2024-11-10T12:28:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T12:28:04,823 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef177eb06ca366a257ef64b1dd9fbfa3, regionState=OPENING, regionLocation=3857ccc89b65,45393,1731241682366 2024-11-10T12:28:04,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=ef177eb06ca366a257ef64b1dd9fbfa3, ASSIGN because future has completed 2024-11-10T12:28:04,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef177eb06ca366a257ef64b1dd9fbfa3, server=3857ccc89b65,45393,1731241682366}] 2024-11-10T12:28:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T12:28:04,983 INFO [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:04,983 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ef177eb06ca366a257ef64b1dd9fbfa3, NAME => 'table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3.', STARTKEY => '', ENDKEY => ''} 2024-11-10T12:28:04,984 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,984 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T12:28:04,984 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,984 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,985 INFO [StoreOpener-ef177eb06ca366a257ef64b1dd9fbfa3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,989 INFO [StoreOpener-ef177eb06ca366a257ef64b1dd9fbfa3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef177eb06ca366a257ef64b1dd9fbfa3 columnFamilyName family 2024-11-10T12:28:04,990 DEBUG [StoreOpener-ef177eb06ca366a257ef64b1dd9fbfa3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T12:28:04,990 INFO [StoreOpener-ef177eb06ca366a257ef64b1dd9fbfa3-1 {}] regionserver.HStore(327): Store=ef177eb06ca366a257ef64b1dd9fbfa3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T12:28:04,991 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,991 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,992 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,992 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,992 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,993 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:04,999 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T12:28:04,999 INFO [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ef177eb06ca366a257ef64b1dd9fbfa3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61239874, jitterRate=-0.08745476603507996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T12:28:04,999 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:05,000 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ef177eb06ca366a257ef64b1dd9fbfa3: Running coprocessor pre-open hook at 1731241684984Writing region info on filesystem at 1731241684984Initializing all the Stores at 1731241684985 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731241684985Cleaning up temporary data from old regions at 1731241684992 (+7 ms)Running coprocessor post-open hooks at 1731241684999 (+7 ms)Region opened successfully at 1731241685000 (+1 ms) 2024-11-10T12:28:05,002 INFO [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3., pid=6, masterSystemTime=1731241684980 2024-11-10T12:28:05,004 DEBUG [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:05,004 INFO [RS_OPEN_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:05,005 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef177eb06ca366a257ef64b1dd9fbfa3, regionState=OPEN, openSeqNum=2, regionLocation=3857ccc89b65,45393,1731241682366 2024-11-10T12:28:05,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef177eb06ca366a257ef64b1dd9fbfa3, server=3857ccc89b65,45393,1731241682366 because future has completed 2024-11-10T12:28:05,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T12:28:05,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ef177eb06ca366a257ef64b1dd9fbfa3, server=3857ccc89b65,45393,1731241682366 in 183 msec 2024-11-10T12:28:05,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T12:28:05,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=ef177eb06ca366a257ef64b1dd9fbfa3, ASSIGN in 344 msec 2024-11-10T12:28:05,015 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T12:28:05,015 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731241685015"}]},"ts":"1731241685015"} 2024-11-10T12:28:05,018 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-10T12:28:05,019 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T12:28:05,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 428 msec 2024-11-10T12:28:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T12:28:05,256 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-10T12:28:05,255 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-10T12:28:05,258 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T12:28:05,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-10T12:28:05,265 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T12:28:05,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-10T12:28:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3., hostname=3857ccc89b65,45393,1731241682366, seqNum=2] 2024-11-10T12:28:05,286 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ef177eb06ca366a257ef64b1dd9fbfa3 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T12:28:05,310 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f3acf05d0e6b405f8f63f4e1a754cc16 2024-11-10T12:28:05,350 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/.tmp/family/f3acf05d0e6b405f8f63f4e1a754cc16 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T12:28:05,355 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/.tmp/family/f3acf05d0e6b405f8f63f4e1a754cc16 2024-11-10T12:28:05,393 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/.tmp/family/f3acf05d0e6b405f8f63f4e1a754cc16 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16 2024-11-10T12:28:05,394 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16, delay=948 ms 2024-11-10T12:28:05,396 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T12:28:05,402 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ef177eb06ca366a257ef64b1dd9fbfa3 in 115ms, sequenceid=6, compaction requested=false 2024-11-10T12:28:05,403 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-10T12:28:05,405 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ef177eb06ca366a257ef64b1dd9fbfa3: 2024-11-10T12:28:06,345 DEBUG [hfile-prefetch-1731241685395 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f3acf05d0e6b405f8f63f4e1a754cc16_0. 2024-11-10T12:28:06,349 DEBUG [hfile-prefetch-1731241685395 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f3acf05d0e6b405f8f63f4e1a754cc16_117. 2024-11-10T12:28:06,350 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16, with total blocks 2, and data blocks 1 2024-11-10T12:28:06,350 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T12:28:06,350 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T12:28:06,351 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16. Will sleep 100ms and try the verification again. 2024-11-10T12:28:06,451 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16, with total blocks 2, and data blocks 1 2024-11-10T12:28:06,451 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-10T12:28:06,451 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2294): found block for file f3acf05d0e6b405f8f63f4e1a754cc16 in the backing map. Acquiring read lock for offset 0 2024-11-10T12:28:06,451 DEBUG [hfile-prefetch-1731241685395 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/family/f3acf05d0e6b405f8f63f4e1a754cc16 has now been fully cached. 2024-11-10T12:28:06,452 DEBUG [hfile-prefetch-1731241685395 {}] hfile.PrefetchExecutor(121): Prefetch completed for f3acf05d0e6b405f8f63f4e1a754cc16 2024-11-10T12:28:06,453 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T12:28:06,453 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T12:28:06,453 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T12:28:06,453 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-10T12:28:06,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-10T12:28:06,474 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731241686453 2024-11-10T12:28:06,474 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T12:28:07,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T12:28:07,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T12:28:07,413 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;3857ccc89b65:45393,5,FailOnTimeoutGroup] 2024-11-10T12:28:07,414 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3857ccc89b65,45393,1731241682366' ***** 2024-11-10T12:28:07,414 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-10T12:28:07,414 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-10T12:28:07,414 INFO [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T12:28:07,414 INFO [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/bucket.persistence 2024-11-10T12:28:07,414 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T12:28:07,414 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T12:28:07,414 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T12:28:07,415 INFO [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T12:28:07,415 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T12:28:07,424 INFO [regionserver/3857ccc89b65:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T12:28:07,424 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T12:28:07,425 DEBUG [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1488): Thread RS:0;3857ccc89b65:45393 started persisting bucket cache to file 2024-11-10T12:28:07,425 DEBUG [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T12:28:07,425 DEBUG [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-10T12:28:07,428 DEBUG [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-10T12:28:07,428 DEBUG [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731241687425 2024-11-10T12:28:07,428 DEBUG [RS:0;3857ccc89b65:45393 {}] bucket.BucketCache(1510): Thread RS:0;3857ccc89b65:45393 finished persisting bucket cache to file, renaming 2024-11-10T12:28:07,429 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T12:28:07,429 INFO [RS:0;3857ccc89b65:45393 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T12:28:07,429 INFO [RS:0;3857ccc89b65:45393 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T12:28:07,429 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:07,430 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(959): stopping server 3857ccc89b65,45393,1731241682366 2024-11-10T12:28:07,430 INFO [RS:0;3857ccc89b65:45393 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T12:28:07,430 INFO [RS:0;3857ccc89b65:45393 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3857ccc89b65:45393. 2024-11-10T12:28:07,430 INFO [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T12:28:07,430 DEBUG [RS:0;3857ccc89b65:45393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T12:28:07,430 DEBUG [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef177eb06ca366a257ef64b1dd9fbfa3, disabling compactions & flushes 2024-11-10T12:28:07,431 INFO [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:07,431 DEBUG [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:07,431 DEBUG [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. after waiting 0 ms 2024-11-10T12:28:07,431 DEBUG [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:07,436 DEBUG [RS:0;3857ccc89b65:45393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T12:28:07,436 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T12:28:07,436 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T12:28:07,436 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T12:28:07,437 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T12:28:07,437 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T12:28:07,437 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1325): Online Regions={ef177eb06ca366a257ef64b1dd9fbfa3=table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T12:28:07,437 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T12:28:07,437 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T12:28:07,437 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T12:28:07,437 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T12:28:07,437 DEBUG [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ef177eb06ca366a257ef64b1dd9fbfa3 2024-11-10T12:28:07,437 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T12:28:07,438 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T12:28:07,438 DEBUG [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/default/table1/ef177eb06ca366a257ef64b1dd9fbfa3/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T12:28:07,438 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-10T12:28:07,438 INFO [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:07,438 DEBUG [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef177eb06ca366a257ef64b1dd9fbfa3: Waiting for close lock at 1731241687430Running coprocessor pre-close hooks at 1731241687430Disabling compacts and flushes for region at 1731241687430Disabling writes for close at 1731241687431 (+1 ms)Writing region close event to WAL at 1731241687432 (+1 ms)Running coprocessor post-close hooks at 1731241687438 (+6 ms)Closed at 1731241687438 2024-11-10T12:28:07,439 DEBUG [RS_CLOSE_REGION-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3. 2024-11-10T12:28:07,451 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6a82e45bf96e4bd28dc2e89da096402a 2024-11-10T12:28:07,471 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/info/6a82e45bf96e4bd28dc2e89da096402a is 133, key is table1,,1731241684584.ef177eb06ca366a257ef64b1dd9fbfa3./info:regioninfo/1731241685005/Put/seqid=0 2024-11-10T12:28:07,472 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/info/6a82e45bf96e4bd28dc2e89da096402a 2024-11-10T12:28:07,483 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 84655a4f24cc48b7a5279f3d9b61d9bb 2024-11-10T12:28:07,496 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/ns/84655a4f24cc48b7a5279f3d9b61d9bb is 43, key is default/ns:d/1731241684327/Put/seqid=0 2024-11-10T12:28:07,497 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/ns/84655a4f24cc48b7a5279f3d9b61d9bb 2024-11-10T12:28:07,505 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b032da2669ce4488a51d535b7354fca6 2024-11-10T12:28:07,519 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/table/b032da2669ce4488a51d535b7354fca6 is 42, key is table1/table:state/1731241685015/Put/seqid=0 2024-11-10T12:28:07,520 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/table/b032da2669ce4488a51d535b7354fca6 2024-11-10T12:28:07,522 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/info/6a82e45bf96e4bd28dc2e89da096402a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/info/6a82e45bf96e4bd28dc2e89da096402a 2024-11-10T12:28:07,524 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/info/6a82e45bf96e4bd28dc2e89da096402a, delay=984 ms 2024-11-10T12:28:07,525 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/info/6a82e45bf96e4bd28dc2e89da096402a, entries=10, sequenceid=11, filesize=6.3 K 2024-11-10T12:28:07,526 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/ns/84655a4f24cc48b7a5279f3d9b61d9bb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/ns/84655a4f24cc48b7a5279f3d9b61d9bb 2024-11-10T12:28:07,527 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/ns/84655a4f24cc48b7a5279f3d9b61d9bb, delay=963 ms 2024-11-10T12:28:07,528 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/ns/84655a4f24cc48b7a5279f3d9b61d9bb, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T12:28:07,528 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/.tmp/table/b032da2669ce4488a51d535b7354fca6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/table/b032da2669ce4488a51d535b7354fca6 2024-11-10T12:28:07,530 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/table/b032da2669ce4488a51d535b7354fca6, delay=989 ms 2024-11-10T12:28:07,531 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/table/b032da2669ce4488a51d535b7354fca6, entries=2, sequenceid=11, filesize=5.1 K 2024-11-10T12:28:07,532 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false 2024-11-10T12:28:07,532 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T12:28:07,533 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/info/6a82e45bf96e4bd28dc2e89da096402a 2024-11-10T12:28:07,534 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/ns/84655a4f24cc48b7a5279f3d9b61d9bb 2024-11-10T12:28:07,535 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/table/b032da2669ce4488a51d535b7354fca6 2024-11-10T12:28:07,539 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T12:28:07,539 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T12:28:07,540 INFO [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T12:28:07,540 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731241687437Running coprocessor pre-close hooks at 1731241687437Disabling compacts and flushes for region at 1731241687437Disabling writes for close at 1731241687437Obtaining lock to block concurrent updates at 1731241687438 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731241687438Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731241687438Flushing stores of hbase:meta,,1.1588230740 at 1731241687439 (+1 ms)Flushing 1588230740/info: creating writer at 1731241687439Flushing 1588230740/info: appending metadata at 1731241687470 (+31 ms)Flushing 1588230740/info: closing flushed file at 1731241687470Flushing 1588230740/ns: creating writer at 1731241687476 (+6 ms)Flushing 1588230740/ns: appending metadata at 1731241687496 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731241687496Flushing 1588230740/table: creating writer at 1731241687499 (+3 ms)Flushing 1588230740/table: appending metadata at 1731241687518 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731241687518Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7089df33: reopening flushed file at 1731241687522 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a0b2a2c: reopening flushed file at 1731241687525 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72f2d73d: reopening flushed file at 1731241687528 (+3 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false at 1731241687532 (+4 ms)Writing region close event to WAL at 1731241687535 (+3 ms)Running coprocessor post-close hooks at 1731241687539 (+4 ms)Closed at 1731241687540 (+1 ms) 2024-11-10T12:28:07,540 DEBUG [RS_CLOSE_META-regionserver/3857ccc89b65:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T12:28:07,612 INFO [regionserver/3857ccc89b65:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T12:28:07,613 INFO [regionserver/3857ccc89b65:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T12:28:07,638 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(976): stopping server 3857ccc89b65,45393,1731241682366; all regions closed. 2024-11-10T12:28:07,643 DEBUG [RS:0;3857ccc89b65:45393 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/oldWALs 2024-11-10T12:28:07,643 INFO [RS:0;3857ccc89b65:45393 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3857ccc89b65%2C45393%2C1731241682366.meta:.meta(num 1731241684192) 2024-11-10T12:28:07,645 DEBUG [RS:0;3857ccc89b65:45393 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/oldWALs 2024-11-10T12:28:07,645 INFO [RS:0;3857ccc89b65:45393 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3857ccc89b65%2C45393%2C1731241682366:(num 1731241683777) 2024-11-10T12:28:07,645 DEBUG [RS:0;3857ccc89b65:45393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T12:28:07,645 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T12:28:07,645 INFO [RS:0;3857ccc89b65:45393 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T12:28:07,646 INFO [RS:0;3857ccc89b65:45393 {}] hbase.ChoreService(370): Chore service for: regionserver/3857ccc89b65:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T12:28:07,646 INFO [RS:0;3857ccc89b65:45393 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T12:28:07,646 INFO [regionserver/3857ccc89b65:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T12:28:07,646 INFO [RS:0;3857ccc89b65:45393 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45393 2024-11-10T12:28:07,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T12:28:07,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3857ccc89b65,45393,1731241682366 2024-11-10T12:28:07,652 INFO [RS:0;3857ccc89b65:45393 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T12:28:07,653 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3857ccc89b65,45393,1731241682366] 2024-11-10T12:28:07,657 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3857ccc89b65,45393,1731241682366 already deleted, retry=false 2024-11-10T12:28:07,657 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3857ccc89b65,45393,1731241682366 on 3857ccc89b65,43729,1731241681587 2024-11-10T12:28:07,662 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3857ccc89b65,45393,1731241682366, splitWal=true, meta=true 2024-11-10T12:28:07,664 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3857ccc89b65,45393,1731241682366, splitWal=true, meta=true 2024-11-10T12:28:07,665 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for 3857ccc89b65,45393,1731241682366 (carryingMeta=true) 3857ccc89b65,45393,1731241682366/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@365fb93d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-10T12:28:07,665 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3857ccc89b65,45393,1731241682366, splitWal=true, meta=true, isMeta: true 2024-11-10T12:28:07,665 DEBUG [PEWorker-3 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366-splitting 2024-11-10T12:28:07,665 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366-splitting dir is empty, no logs to split. 2024-11-10T12:28:07,666 INFO [PEWorker-3 {}] master.SplitWALManager(105): 3857ccc89b65,45393,1731241682366 WAL count=0, meta=true 2024-11-10T12:28:07,667 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/WALs/3857ccc89b65,45393,1731241682366-splitting dir is empty, no logs to split. 2024-11-10T12:28:07,667 INFO [PEWorker-3 {}] master.SplitWALManager(105): 3857ccc89b65,45393,1731241682366 WAL count=0, meta=true 2024-11-10T12:28:07,667 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 3857ccc89b65,45393,1731241682366 WAL splitting is done? wals=0, meta=true 2024-11-10T12:28:07,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T12:28:07,670 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T12:28:07,671 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-10T12:28:07,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T12:28:07,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x10101d98c100001, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T12:28:07,756 INFO [RS:0;3857ccc89b65:45393 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T12:28:07,756 INFO [RS:0;3857ccc89b65:45393 {}] regionserver.HRegionServer(1031): Exiting; stopping=3857ccc89b65,45393,1731241682366; zookeeper connection closed. 2024-11-10T12:28:07,757 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@40525ea6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@40525ea6 2024-11-10T12:28:07,822 WARN [3857ccc89b65:43729 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T12:28:08,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T12:28:08,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T12:28:08,415 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T12:28:08,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T12:28:08,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T12:28:08,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T12:28:08,416 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T12:28:08,416 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1590087585, stopped=false 2024-11-10T12:28:08,416 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3857ccc89b65,43729,1731241681587 2024-11-10T12:28:08,416 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '3857ccc89b65,43729,1731241681587' ***** 2024-11-10T12:28:08,416 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-10T12:28:08,416 INFO [M:0;3857ccc89b65:43729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T12:28:08,417 INFO [M:0;3857ccc89b65:43729 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;3857ccc89b65:43729. 2024-11-10T12:28:08,417 DEBUG [M:0;3857ccc89b65:43729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T12:28:08,417 DEBUG [M:0;3857ccc89b65:43729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T12:28:08,417 INFO [M:0;3857ccc89b65:43729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T12:28:08,417 DEBUG [M:0;3857ccc89b65:43729 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T12:28:08,417 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T12:28:08,417 DEBUG [M:0;3857ccc89b65:43729 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T12:28:08,417 DEBUG [master/3857ccc89b65:0:becomeActiveMaster-HFileCleaner.small.0-1731241683465 {}] cleaner.HFileCleaner(306): Exit Thread[master/3857ccc89b65:0:becomeActiveMaster-HFileCleaner.small.0-1731241683465,5,FailOnTimeoutGroup] 2024-11-10T12:28:08,417 DEBUG [master/3857ccc89b65:0:becomeActiveMaster-HFileCleaner.large.0-1731241683464 {}] cleaner.HFileCleaner(306): Exit Thread[master/3857ccc89b65:0:becomeActiveMaster-HFileCleaner.large.0-1731241683464,5,FailOnTimeoutGroup] 2024-11-10T12:28:08,418 INFO [M:0;3857ccc89b65:43729 {}] hbase.ChoreService(370): Chore service for: master/3857ccc89b65:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T12:28:08,418 INFO [M:0;3857ccc89b65:43729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T12:28:08,418 DEBUG [M:0;3857ccc89b65:43729 {}] master.HMaster(1795): Stopping service threads 2024-11-10T12:28:08,418 INFO [M:0;3857ccc89b65:43729 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T12:28:08,418 INFO [M:0;3857ccc89b65:43729 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T12:28:08,419 INFO [M:0;3857ccc89b65:43729 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T12:28:08,419 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T12:28:08,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T12:28:08,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T12:28:08,420 DEBUG [M:0;3857ccc89b65:43729 {}] zookeeper.ZKUtil(347): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T12:28:08,420 WARN [M:0;3857ccc89b65:43729 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T12:28:08,421 INFO [M:0;3857ccc89b65:43729 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/.lastflushedseqids 2024-11-10T12:28:08,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T12:28:08,422 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T12:28:08,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T12:28:08,434 INFO [M:0;3857ccc89b65:43729 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T12:28:08,572 DEBUG [3857ccc89b65:43729 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-10T12:28:08,572 INFO [M:0;3857ccc89b65:43729 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T12:28:08,573 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T12:28:08,573 INFO [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:08,573 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:08,573 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T12:28:08,573 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:08,573 INFO [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-11-10T12:28:08,584 DEBUG [M:0;3857ccc89b65:43729 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5411b28731a440c6a5d4a97220861ed4 2024-11-10T12:28:08,599 DEBUG [M:0;3857ccc89b65:43729 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5411b28731a440c6a5d4a97220861ed4 is 82, key is hbase:meta,,1/info:regioninfo/1731241684261/Put/seqid=0 2024-11-10T12:28:08,600 INFO [M:0;3857ccc89b65:43729 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5411b28731a440c6a5d4a97220861ed4 2024-11-10T12:28:08,609 DEBUG [M:0;3857ccc89b65:43729 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 87a9a703410a49dc8cee4e06b9adfed0 2024-11-10T12:28:08,623 DEBUG [M:0;3857ccc89b65:43729 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/87a9a703410a49dc8cee4e06b9adfed0 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731241685020/Put/seqid=0 2024-11-10T12:28:08,624 INFO [M:0;3857ccc89b65:43729 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/87a9a703410a49dc8cee4e06b9adfed0 2024-11-10T12:28:08,632 DEBUG [M:0;3857ccc89b65:43729 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 92589251d1674087b8ff58bc7de09b6c 2024-11-10T12:28:08,647 DEBUG [M:0;3857ccc89b65:43729 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92589251d1674087b8ff58bc7de09b6c is 58, key is 3857ccc89b65,45393,1731241682366/rs:/1731241687666/DeleteFamily/seqid=0 2024-11-10T12:28:08,647 INFO [M:0;3857ccc89b65:43729 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92589251d1674087b8ff58bc7de09b6c 2024-11-10T12:28:08,649 INFO [M:0;3857ccc89b65:43729 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92589251d1674087b8ff58bc7de09b6c 2024-11-10T12:28:08,650 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5411b28731a440c6a5d4a97220861ed4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5411b28731a440c6a5d4a97220861ed4 2024-11-10T12:28:08,652 INFO [M:0;3857ccc89b65:43729 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5411b28731a440c6a5d4a97220861ed4, entries=8, sequenceid=64, filesize=5.5 K 2024-11-10T12:28:08,652 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/87a9a703410a49dc8cee4e06b9adfed0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/87a9a703410a49dc8cee4e06b9adfed0 2024-11-10T12:28:08,654 INFO [M:0;3857ccc89b65:43729 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/87a9a703410a49dc8cee4e06b9adfed0, entries=8, sequenceid=64, filesize=6.5 K 2024-11-10T12:28:08,654 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92589251d1674087b8ff58bc7de09b6c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92589251d1674087b8ff58bc7de09b6c 2024-11-10T12:28:08,655 INFO [M:0;3857ccc89b65:43729 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92589251d1674087b8ff58bc7de09b6c 2024-11-10T12:28:08,656 INFO [M:0;3857ccc89b65:43729 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/685f3415-37a6-4bbc-b4f3-4df42507a669/fbbf2cbe-7241-73f4-3099-a73206d732c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92589251d1674087b8ff58bc7de09b6c, entries=1, sequenceid=64, filesize=5.2 K 2024-11-10T12:28:08,656 INFO [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=64, compaction requested=false 2024-11-10T12:28:08,659 INFO [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T12:28:08,659 DEBUG [M:0;3857ccc89b65:43729 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731241688573Disabling compacts and flushes for region at 1731241688573Disabling writes for close at 1731241688573Obtaining lock to block concurrent updates at 1731241688573Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731241688573Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731241688574 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731241688574Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731241688574Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731241688599 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731241688599Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731241688602 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731241688623 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731241688623Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731241688626 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731241688646 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731241688646Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18b1b7c2: reopening flushed file at 1731241688650 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@438ac082: reopening flushed file at 1731241688652 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a06bb1: reopening flushed file at 1731241688654 (+2 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=64, compaction requested=false at 1731241688657 (+3 ms)Writing region close event to WAL at 1731241688659 (+2 ms)Closed at 1731241688659 2024-11-10T12:28:08,661 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T12:28:08,661 INFO [M:0;3857ccc89b65:43729 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T12:28:08,661 INFO [M:0;3857ccc89b65:43729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43729 2024-11-10T12:28:08,661 INFO [M:0;3857ccc89b65:43729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T12:28:08,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T12:28:08,764 INFO [M:0;3857ccc89b65:43729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T12:28:08,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x10101d98c100000, quorum=127.0.0.1:50874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T12:28:08,768 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T12:28:08,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T12:28:08,791 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T12:28:08,799 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3857ccc89b65:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731241687524 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731241687530 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731241685395 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3857ccc89b65:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731241687527 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3857ccc89b65:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=265 (was 286), ProcessCount=11 (was 11), AvailableMemoryMB=8980 (was 9317)