2024-11-18 03:48:22,569 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-18 03:48:22,594 main DEBUG Took 0.010953 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 03:48:22,594 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 03:48:22,594 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 03:48:22,595 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 03:48:22,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,605 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 03:48:22,619 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,621 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,623 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,624 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,624 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,626 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,627 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,628 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,629 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,630 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,631 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,631 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:22,632 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,632 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 03:48:22,634 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:22,635 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 03:48:22,637 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 03:48:22,637 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 03:48:22,639 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 03:48:22,639 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 03:48:22,649 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 03:48:22,652 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 03:48:22,653 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 03:48:22,654 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 03:48:22,654 main DEBUG createAppenders(={Console}) 2024-11-18 03:48:22,655 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-18 03:48:22,655 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-18 03:48:22,655 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-18 03:48:22,656 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 03:48:22,656 main DEBUG OutputStream closed 2024-11-18 03:48:22,656 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 03:48:22,656 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 03:48:22,657 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-18 03:48:22,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 03:48:22,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 03:48:22,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 03:48:22,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 03:48:22,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 03:48:22,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 03:48:22,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 03:48:22,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 03:48:22,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 03:48:22,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 03:48:22,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 03:48:22,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 03:48:22,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 03:48:22,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 03:48:22,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 03:48:22,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 03:48:22,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 03:48:22,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 03:48:22,758 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 03:48:22,758 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-18 03:48:22,759 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 03:48:22,760 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-18T03:48:22,779 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-18 03:48:22,783 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 03:48:22,784 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T03:48:23,067 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f 2024-11-18T03:48:23,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=609, ProcessCount=11, AvailableMemoryMB=6638 2024-11-18T03:48:23,239 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T03:48:23,384 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:23,417 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/cluster_6fb194d0-7d31-3951-c663-8a56e992bec0, deleteOnExit=true 2024-11-18T03:48:23,546 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/cluster_6fb194d0-7d31-3951-c663-8a56e992bec0/zookeeper_0, clientPort=64646, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/cluster_6fb194d0-7d31-3951-c663-8a56e992bec0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/cluster_6fb194d0-7d31-3951-c663-8a56e992bec0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T03:48:23,563 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64646 2024-11-18T03:48:23,564 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:23,719 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084 with version=8 2024-11-18T03:48:23,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/hbase-staging 2024-11-18T03:48:23,849 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T03:48:24,179 INFO [Time-limited test {}] client.ConnectionUtils(128): master/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:24,194 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:24,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:24,208 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:24,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:24,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:24,391 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T03:48:24,499 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:48:24,561 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T03:48:24,576 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T03:48:24,581 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:24,639 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21902 (auto-detected) 2024-11-18T03:48:24,641 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T03:48:24,672 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38265 2024-11-18T03:48:24,705 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38265 connecting to ZooKeeper ensemble=127.0.0.1:64646 2024-11-18T03:48:24,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:382650x0, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:24,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38265-0x1012930afe20000 connected 2024-11-18T03:48:24,809 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:24,810 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:24,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:24,826 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084, hbase.cluster.distributed=false 2024-11-18T03:48:24,856 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:24,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38265 2024-11-18T03:48:24,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38265 2024-11-18T03:48:24,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38265 2024-11-18T03:48:24,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38265 2024-11-18T03:48:24,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38265 2024-11-18T03:48:24,977 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:24,978 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:24,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:24,979 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:24,980 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:24,980 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:24,984 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T03:48:24,988 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:24,989 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33631 2024-11-18T03:48:24,991 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33631 connecting to ZooKeeper ensemble=127.0.0.1:64646 2024-11-18T03:48:24,991 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:24,992 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:24,999 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336310x0, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:25,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:336310x0, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:25,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33631-0x1012930afe20001 connected 2024-11-18T03:48:25,008 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T03:48:25,029 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/bucket.cache 2024-11-18T03:48:25,037 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T03:48:25,039 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T03:48:25,041 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-18T03:48:25,043 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T03:48:25,044 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T03:48:25,051 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T03:48:25,056 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T03:48:25,056 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/bucket.persistence is complete. 2024-11-18T03:48:25,059 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T03:48:25,062 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T03:48:25,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:25,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33631 2024-11-18T03:48:25,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33631 2024-11-18T03:48:25,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33631 2024-11-18T03:48:25,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33631 2024-11-18T03:48:25,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33631 2024-11-18T03:48:25,097 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;88c1dcece5ca:38265 2024-11-18T03:48:25,098 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:25,105 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:25,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:25,108 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:25,140 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:25,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:25,140 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:25,142 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T03:48:25,144 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/88c1dcece5ca,38265,1731901703942 from backup master directory 2024-11-18T03:48:25,148 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:25,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:25,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:25,149 WARN [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:25,150 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:25,152 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T03:48:25,154 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T03:48:25,164 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/hbase.id] with ID: 69a33ff7-0db8-41b5-9547-b129a2f62dd4 2024-11-18T03:48:25,164 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/.tmp/hbase.id 2024-11-18T03:48:25,174 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/hbase.id] 2024-11-18T03:48:25,237 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:25,244 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T03:48:25,264 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-18T03:48:25,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:25,269 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:25,301 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:48:25,306 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T03:48:25,364 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:48:25,372 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:25,445 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store 2024-11-18T03:48:25,493 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T03:48:25,498 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:25,500 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:48:25,500 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:25,500 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:25,502 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:48:25,503 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:25,503 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:25,504 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731901705499Disabling compacts and flushes for region at 1731901705499Disabling writes for close at 1731901705503 (+4 ms)Writing region close event to WAL at 1731901705503Closed at 1731901705503 2024-11-18T03:48:25,505 WARN [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/.initializing 2024-11-18T03:48:25,505 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/WALs/88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:25,535 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:25,560 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88c1dcece5ca%2C38265%2C1731901703942, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/WALs/88c1dcece5ca,38265,1731901703942, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/oldWALs, maxLogs=10 2024-11-18T03:48:25,631 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/WALs/88c1dcece5ca,38265,1731901703942/88c1dcece5ca%2C38265%2C1731901703942.1731901705566 2024-11-18T03:48:25,633 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:25,633 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:25,634 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:25,639 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,640 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T03:48:25,756 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:25,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:25,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T03:48:25,767 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:25,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:25,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,774 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T03:48:25,775 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:25,775 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:25,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,782 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T03:48:25,782 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:25,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:25,783 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,788 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,789 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,793 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,794 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,799 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:25,804 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:25,833 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:25,835 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69404279, jitterRate=0.03420434892177582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:25,843 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731901705656Initializing all the Stores at 1731901705657 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901705657Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901705665 (+8 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901705666 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901705666Cleaning up temporary data from old regions at 1731901705795 (+129 ms)Region opened successfully at 1731901705843 (+48 ms) 2024-11-18T03:48:25,845 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T03:48:25,886 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f172e15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:25,926 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T03:48:25,938 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T03:48:25,939 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T03:48:25,942 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T03:48:25,943 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T03:48:25,950 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-18T03:48:25,950 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T03:48:25,981 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T03:48:25,991 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T03:48:25,993 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T03:48:25,996 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T03:48:25,997 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T03:48:25,999 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T03:48:26,001 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T03:48:26,005 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T03:48:26,008 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T03:48:26,010 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T03:48:26,011 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T03:48:26,034 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T03:48:26,036 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T03:48:26,050 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:26,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:26,051 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,054 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=88c1dcece5ca,38265,1731901703942, sessionid=0x1012930afe20000, setting cluster-up flag (Was=false) 2024-11-18T03:48:26,068 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,081 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T03:48:26,084 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:26,089 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,099 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T03:48:26,102 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:26,107 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T03:48:26,184 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(746): ClusterId : 69a33ff7-0db8-41b5-9547-b129a2f62dd4 2024-11-18T03:48:26,188 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T03:48:26,194 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T03:48:26,194 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T03:48:26,203 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:26,206 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T03:48:26,207 DEBUG [RS:0;88c1dcece5ca:33631 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273ae012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:26,220 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T03:48:26,229 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;88c1dcece5ca:33631 2024-11-18T03:48:26,230 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T03:48:26,234 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T03:48:26,235 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T03:48:26,235 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T03:48:26,239 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(2659): reportForDuty to master=88c1dcece5ca,38265,1731901703942 with port=33631, startcode=1731901704936 2024-11-18T03:48:26,237 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 88c1dcece5ca,38265,1731901703942 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T03:48:26,246 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:26,246 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:26,246 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:26,247 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:26,247 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/88c1dcece5ca:0, corePoolSize=10, maxPoolSize=10 2024-11-18T03:48:26,247 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,248 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:26,248 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,257 DEBUG [RS:0;88c1dcece5ca:33631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T03:48:26,257 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731901736257 2024-11-18T03:48:26,259 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T03:48:26,261 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T03:48:26,261 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:26,262 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T03:48:26,264 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T03:48:26,265 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T03:48:26,265 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T03:48:26,265 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T03:48:26,268 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:26,268 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T03:48:26,268 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,272 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T03:48:26,274 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T03:48:26,274 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T03:48:26,288 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T03:48:26,289 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T03:48:26,294 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901706290,5,FailOnTimeoutGroup] 2024-11-18T03:48:26,295 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901706294,5,FailOnTimeoutGroup] 2024-11-18T03:48:26,295 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,296 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T03:48:26,297 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,297 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,321 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T03:48:26,322 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084 2024-11-18T03:48:26,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:26,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T03:48:26,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:48:26,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:26,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:26,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T03:48:26,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:48:26,354 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:26,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37027, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T03:48:26,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:26,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T03:48:26,362 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38265 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:26,365 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38265 {}] master.ServerManager(517): Registering regionserver=88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:26,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:48:26,367 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:26,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:26,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T03:48:26,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:48:26,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:26,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:26,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:48:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740 2024-11-18T03:48:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740 2024-11-18T03:48:26,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:48:26,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:48:26,387 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084 2024-11-18T03:48:26,387 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T03:48:26,387 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:26,387 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T03:48:26,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:48:26,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:26,394 DEBUG [RS:0;88c1dcece5ca:33631 {}] zookeeper.ZKUtil(111): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:26,394 WARN [RS:0;88c1dcece5ca:33631 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:26,394 INFO [RS:0;88c1dcece5ca:33631 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:26,395 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:26,401 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:26,401 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63960348, jitterRate=-0.04691654443740845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:26,402 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88c1dcece5ca,33631,1731901704936] 2024-11-18T03:48:26,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731901706335Initializing all the Stores at 1731901706335Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901706335Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901706336 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901706336Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901706336Cleaning up temporary data from old regions at 1731901706386 (+50 ms)Region opened successfully at 1731901706405 (+19 ms) 2024-11-18T03:48:26,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:48:26,406 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:48:26,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:48:26,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:48:26,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:48:26,408 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:26,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731901706405Disabling compacts and flushes for region at 1731901706405Disabling writes for close at 1731901706406 (+1 ms)Writing region close event to WAL at 1731901706408 (+2 ms)Closed at 1731901706408 2024-11-18T03:48:26,411 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:26,411 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T03:48:26,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T03:48:26,426 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T03:48:26,430 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:48:26,433 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T03:48:26,453 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T03:48:26,459 INFO [RS:0;88c1dcece5ca:33631 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T03:48:26,460 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,461 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T03:48:26,468 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T03:48:26,470 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,471 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,471 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,471 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,471 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,472 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,472 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:26,472 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,472 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,473 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,473 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,473 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,473 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:26,474 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:26,474 DEBUG [RS:0;88c1dcece5ca:33631 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:26,485 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,485 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,485 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,486 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,486 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,486 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,33631,1731901704936-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:26,508 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T03:48:26,510 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,33631,1731901704936-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,511 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,511 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.Replication(171): 88c1dcece5ca,33631,1731901704936 started 2024-11-18T03:48:26,534 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:26,535 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1482): Serving as 88c1dcece5ca,33631,1731901704936, RpcServer on 88c1dcece5ca/172.17.0.2:33631, sessionid=0x1012930afe20001 2024-11-18T03:48:26,535 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T03:48:26,536 DEBUG [RS:0;88c1dcece5ca:33631 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:26,536 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,33631,1731901704936' 2024-11-18T03:48:26,536 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T03:48:26,538 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T03:48:26,539 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T03:48:26,539 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T03:48:26,539 DEBUG [RS:0;88c1dcece5ca:33631 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:26,539 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,33631,1731901704936' 2024-11-18T03:48:26,539 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T03:48:26,540 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T03:48:26,541 DEBUG [RS:0;88c1dcece5ca:33631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T03:48:26,542 INFO [RS:0;88c1dcece5ca:33631 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T03:48:26,542 INFO [RS:0;88c1dcece5ca:33631 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T03:48:26,584 WARN [88c1dcece5ca:38265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T03:48:26,647 INFO [RS:0;88c1dcece5ca:33631 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:26,648 INFO [RS:0;88c1dcece5ca:33631 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C33631%2C1731901704936, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/oldWALs, maxLogs=55 2024-11-18T03:48:26,679 INFO [RS:0;88c1dcece5ca:33631 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936/88c1dcece5ca%2C33631%2C1731901704936.1731901706651 2024-11-18T03:48:26,679 DEBUG [RS:0;88c1dcece5ca:33631 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:26,837 DEBUG [88c1dcece5ca:38265 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T03:48:26,851 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:26,856 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88c1dcece5ca,33631,1731901704936, state=OPENING 2024-11-18T03:48:26,861 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T03:48:26,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,863 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:26,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:26,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:26,865 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:48:26,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=88c1dcece5ca,33631,1731901704936}] 2024-11-18T03:48:27,040 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T03:48:27,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53761, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T03:48:27,050 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T03:48:27,051 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:27,051 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T03:48:27,053 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C33631%2C1731901704936.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/oldWALs, maxLogs=55 2024-11-18T03:48:27,073 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936/88c1dcece5ca%2C33631%2C1731901704936.meta.1731901707054.meta 2024-11-18T03:48:27,073 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:27,074 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:27,075 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T03:48:27,078 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T03:48:27,082 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T03:48:27,086 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T03:48:27,087 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:27,087 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T03:48:27,087 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T03:48:27,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T03:48:27,090 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:48:27,090 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:27,090 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:27,090 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T03:48:27,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:48:27,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:27,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:27,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T03:48:27,092 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:48:27,092 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:27,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:27,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T03:48:27,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:48:27,094 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:27,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:27,094 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:48:27,095 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740 2024-11-18T03:48:27,096 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740 2024-11-18T03:48:27,096 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:48:27,096 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:48:27,097 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:27,098 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:48:27,098 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67080554, jitterRate=-4.2185187339782715E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:27,099 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T03:48:27,100 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731901707088Writing region info on filesystem at 1731901707088Initializing all the Stores at 1731901707088Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901707088Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901707088Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901707088Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901707089 (+1 ms)Cleaning up temporary data from old regions at 1731901707096 (+7 ms)Running coprocessor post-open hooks at 1731901707099 (+3 ms)Region opened successfully at 1731901707100 (+1 ms) 2024-11-18T03:48:27,106 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731901707031 2024-11-18T03:48:27,116 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T03:48:27,116 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T03:48:27,117 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:27,118 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88c1dcece5ca,33631,1731901704936, state=OPEN 2024-11-18T03:48:27,127 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:27,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:27,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:27,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:27,127 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:27,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T03:48:27,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88c1dcece5ca,33631,1731901704936 in 260 msec 2024-11-18T03:48:27,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T03:48:27,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 713 msec 2024-11-18T03:48:27,135 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:27,135 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T03:48:27,156 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:48:27,157 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88c1dcece5ca,33631,1731901704936, seqNum=-1] 2024-11-18T03:48:27,176 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:48:27,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:48:27,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0500 sec 2024-11-18T03:48:27,195 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731901707195, completionTime=-1 2024-11-18T03:48:27,198 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T03:48:27,198 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T03:48:27,228 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-18T03:48:27,229 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731901767229 2024-11-18T03:48:27,229 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731901827229 2024-11-18T03:48:27,229 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-18T03:48:27,231 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,38265,1731901703942-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:27,232 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,38265,1731901703942-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:27,232 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,38265,1731901703942-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:27,233 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-88c1dcece5ca:38265, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:27,234 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:27,235 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:27,240 DEBUG [master/88c1dcece5ca:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T03:48:27,259 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.109sec 2024-11-18T03:48:27,261 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T03:48:27,262 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T03:48:27,263 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T03:48:27,263 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T03:48:27,263 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T03:48:27,264 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,38265,1731901703942-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:27,264 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,38265,1731901703942-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T03:48:27,268 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T03:48:27,269 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T03:48:27,269 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,38265,1731901703942-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:27,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195e248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:27,294 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T03:48:27,295 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T03:48:27,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 88c1dcece5ca,38265,-1 for getting cluster id 2024-11-18T03:48:27,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:48:27,309 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '69a33ff7-0db8-41b5-9547-b129a2f62dd4' 2024-11-18T03:48:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:48:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "69a33ff7-0db8-41b5-9547-b129a2f62dd4" 2024-11-18T03:48:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2178cf05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [88c1dcece5ca,38265,-1] 2024-11-18T03:48:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:48:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:27,318 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:48:27,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a63dbd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:27,322 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:48:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88c1dcece5ca,33631,1731901704936, seqNum=-1] 2024-11-18T03:48:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:48:27,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:48:27,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:27,376 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:48:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@41fb976 2024-11-18T03:48:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:48:27,386 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:48:27,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:48:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38265 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-18T03:48:27,398 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:48:27,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-18T03:48:27,400 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:27,402 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:48:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:27,415 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 44caa23bff6d6eba4f6dfb7af92d2fe4, NAME => 'table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084 2024-11-18T03:48:27,421 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:27,421 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 44caa23bff6d6eba4f6dfb7af92d2fe4, disabling compactions & flushes 2024-11-18T03:48:27,422 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:27,422 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:27,422 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. after waiting 0 ms 2024-11-18T03:48:27,422 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:27,422 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:27,422 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 44caa23bff6d6eba4f6dfb7af92d2fe4: Waiting for close lock at 1731901707421Disabling compacts and flushes for region at 1731901707421Disabling writes for close at 1731901707422 (+1 ms)Writing region close event to WAL at 1731901707422Closed at 1731901707422 2024-11-18T03:48:27,423 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:48:27,428 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731901707423"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731901707423"}]},"ts":"1731901707423"} 2024-11-18T03:48:27,432 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:48:27,433 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:48:27,435 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731901707433"}]},"ts":"1731901707433"} 2024-11-18T03:48:27,439 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-18T03:48:27,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=44caa23bff6d6eba4f6dfb7af92d2fe4, ASSIGN}] 2024-11-18T03:48:27,442 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=44caa23bff6d6eba4f6dfb7af92d2fe4, ASSIGN 2024-11-18T03:48:27,443 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=44caa23bff6d6eba4f6dfb7af92d2fe4, ASSIGN; state=OFFLINE, location=88c1dcece5ca,33631,1731901704936; forceNewPlan=false, retain=false 2024-11-18T03:48:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:27,594 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=44caa23bff6d6eba4f6dfb7af92d2fe4, regionState=OPENING, regionLocation=88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:27,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=44caa23bff6d6eba4f6dfb7af92d2fe4, ASSIGN because future has completed 2024-11-18T03:48:27,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44caa23bff6d6eba4f6dfb7af92d2fe4, server=88c1dcece5ca,33631,1731901704936}] 2024-11-18T03:48:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:27,754 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:27,755 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 44caa23bff6d6eba4f6dfb7af92d2fe4, NAME => 'table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:27,755 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,755 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:27,755 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,756 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,756 INFO [StoreOpener-44caa23bff6d6eba4f6dfb7af92d2fe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,760 INFO [StoreOpener-44caa23bff6d6eba4f6dfb7af92d2fe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44caa23bff6d6eba4f6dfb7af92d2fe4 columnFamilyName family 2024-11-18T03:48:27,760 DEBUG [StoreOpener-44caa23bff6d6eba4f6dfb7af92d2fe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:27,761 INFO [StoreOpener-44caa23bff6d6eba4f6dfb7af92d2fe4-1 {}] regionserver.HStore(327): Store=44caa23bff6d6eba4f6dfb7af92d2fe4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:27,761 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,762 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,762 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,762 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,762 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,763 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,769 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:27,770 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 44caa23bff6d6eba4f6dfb7af92d2fe4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65715488, jitterRate=-0.020762920379638672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T03:48:27,770 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:27,771 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 44caa23bff6d6eba4f6dfb7af92d2fe4: Running coprocessor pre-open hook at 1731901707756Writing region info on filesystem at 1731901707756Initializing all the Stores at 1731901707756Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901707756Cleaning up temporary data from old regions at 1731901707762 (+6 ms)Running coprocessor post-open hooks at 1731901707770 (+8 ms)Region opened successfully at 1731901707771 (+1 ms) 2024-11-18T03:48:27,772 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4., pid=6, masterSystemTime=1731901707751 2024-11-18T03:48:27,774 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:27,775 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:27,775 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=44caa23bff6d6eba4f6dfb7af92d2fe4, regionState=OPEN, openSeqNum=2, regionLocation=88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:27,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44caa23bff6d6eba4f6dfb7af92d2fe4, server=88c1dcece5ca,33631,1731901704936 because future has completed 2024-11-18T03:48:27,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T03:48:27,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 44caa23bff6d6eba4f6dfb7af92d2fe4, server=88c1dcece5ca,33631,1731901704936 in 180 msec 2024-11-18T03:48:27,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T03:48:27,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=44caa23bff6d6eba4f6dfb7af92d2fe4, ASSIGN in 341 msec 2024-11-18T03:48:27,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:48:27,785 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731901707784"}]},"ts":"1731901707784"} 2024-11-18T03:48:27,787 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-18T03:48:27,788 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:48:27,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 395 msec 2024-11-18T03:48:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:28,037 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-18T03:48:28,037 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-18T03:48:28,040 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T03:48:28,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-18T03:48:28,045 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T03:48:28,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-18T03:48:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4., hostname=88c1dcece5ca,33631,1731901704936, seqNum=2] 2024-11-18T03:48:28,064 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 44caa23bff6d6eba4f6dfb7af92d2fe4 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T03:48:28,088 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 15a9dc7874d0450880e45ccf35f5e915 2024-11-18T03:48:28,124 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/.tmp/family/15a9dc7874d0450880e45ccf35f5e915 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T03:48:28,129 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/.tmp/family/15a9dc7874d0450880e45ccf35f5e915 2024-11-18T03:48:28,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/.tmp/family/15a9dc7874d0450880e45ccf35f5e915 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915 2024-11-18T03:48:28,166 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915, delay=972 ms 2024-11-18T03:48:28,168 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T03:48:28,174 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 44caa23bff6d6eba4f6dfb7af92d2fe4 in 110ms, sequenceid=6, compaction requested=false 2024-11-18T03:48:28,175 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-18T03:48:28,177 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 44caa23bff6d6eba4f6dfb7af92d2fe4: 2024-11-18T03:48:29,141 DEBUG [hfile-prefetch-1731901708166 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 15a9dc7874d0450880e45ccf35f5e915_0. 2024-11-18T03:48:29,145 DEBUG [hfile-prefetch-1731901708166 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 15a9dc7874d0450880e45ccf35f5e915_117. 2024-11-18T03:48:29,147 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915, with total blocks 2, and data blocks 1 2024-11-18T03:48:29,147 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T03:48:29,147 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T03:48:29,148 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915. Will sleep 100ms and try the verification again. 2024-11-18T03:48:29,248 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915, with total blocks 2, and data blocks 1 2024-11-18T03:48:29,248 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-18T03:48:29,248 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2293): found block for file 15a9dc7874d0450880e45ccf35f5e915 in the backing map. Acquiring read lock for offset 0 2024-11-18T03:48:29,248 DEBUG [hfile-prefetch-1731901708166 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/family/15a9dc7874d0450880e45ccf35f5e915 has now been fully cached. 2024-11-18T03:48:29,249 DEBUG [hfile-prefetch-1731901708166 {}] hfile.PrefetchExecutor(121): Prefetch completed for 15a9dc7874d0450880e45ccf35f5e915 2024-11-18T03:48:30,042 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T03:48:30,042 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T03:48:30,043 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:30,043 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-18T03:48:30,062 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-18T03:48:30,062 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731901710042 2024-11-18T03:48:30,062 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T03:48:30,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T03:48:30,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T03:48:30,185 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;88c1dcece5ca:33631,5,FailOnTimeoutGroup] 2024-11-18T03:48:30,185 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88c1dcece5ca,33631,1731901704936' ***** 2024-11-18T03:48:30,186 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-18T03:48:30,186 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-18T03:48:30,186 INFO [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T03:48:30,186 INFO [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/bucket.persistence 2024-11-18T03:48:30,186 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T03:48:30,186 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T03:48:30,186 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T03:48:30,186 INFO [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T03:48:30,187 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T03:48:30,188 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T03:48:30,190 INFO [regionserver/88c1dcece5ca:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:30,197 DEBUG [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1487): Thread RS:0;88c1dcece5ca:33631 started persisting bucket cache to file 2024-11-18T03:48:30,197 DEBUG [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:30,197 DEBUG [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-18T03:48:30,200 DEBUG [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-18T03:48:30,200 DEBUG [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731901710197 2024-11-18T03:48:30,200 DEBUG [RS:0;88c1dcece5ca:33631 {}] bucket.BucketCache(1509): Thread RS:0;88c1dcece5ca:33631 finished persisting bucket cache to file, renaming 2024-11-18T03:48:30,201 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T03:48:30,201 INFO [RS:0;88c1dcece5ca:33631 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T03:48:30,201 INFO [RS:0;88c1dcece5ca:33631 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T03:48:30,201 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(3091): Received CLOSE for 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:30,202 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(959): stopping server 88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:30,202 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:30,202 INFO [RS:0;88c1dcece5ca:33631 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;88c1dcece5ca:33631. 2024-11-18T03:48:30,202 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T03:48:30,202 DEBUG [RS:0;88c1dcece5ca:33631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:30,202 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44caa23bff6d6eba4f6dfb7af92d2fe4, disabling compactions & flushes 2024-11-18T03:48:30,203 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:30,203 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:30,203 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. after waiting 0 ms 2024-11-18T03:48:30,203 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:30,207 DEBUG [RS:0;88c1dcece5ca:33631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:30,207 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T03:48:30,207 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T03:48:30,207 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T03:48:30,208 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T03:48:30,208 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T03:48:30,208 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T03:48:30,208 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 44caa23bff6d6eba4f6dfb7af92d2fe4=table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4.} 2024-11-18T03:48:30,209 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:48:30,209 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:48:30,209 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:48:30,209 DEBUG [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 44caa23bff6d6eba4f6dfb7af92d2fe4 2024-11-18T03:48:30,209 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:48:30,209 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:48:30,209 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/default/table1/44caa23bff6d6eba4f6dfb7af92d2fe4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T03:48:30,209 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-18T03:48:30,210 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:30,210 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44caa23bff6d6eba4f6dfb7af92d2fe4: Waiting for close lock at 1731901710202Running coprocessor pre-close hooks at 1731901710202Disabling compacts and flushes for region at 1731901710202Disabling writes for close at 1731901710203 (+1 ms)Writing region close event to WAL at 1731901710204 (+1 ms)Running coprocessor post-close hooks at 1731901710209 (+5 ms)Closed at 1731901710210 (+1 ms) 2024-11-18T03:48:30,211 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4. 2024-11-18T03:48:30,221 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bcc85b146f2f4b7aa392bf6651ac4364 2024-11-18T03:48:30,238 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/info/bcc85b146f2f4b7aa392bf6651ac4364 is 133, key is table1,,1731901707387.44caa23bff6d6eba4f6dfb7af92d2fe4./info:regioninfo/1731901707775/Put/seqid=0 2024-11-18T03:48:30,239 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/info/bcc85b146f2f4b7aa392bf6651ac4364 2024-11-18T03:48:30,249 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cbb4f7f8e5774d7fbbb95c3ba4016627 2024-11-18T03:48:30,265 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/ns/cbb4f7f8e5774d7fbbb95c3ba4016627 is 43, key is default/ns:d/1731901707183/Put/seqid=0 2024-11-18T03:48:30,265 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/ns/cbb4f7f8e5774d7fbbb95c3ba4016627 2024-11-18T03:48:30,274 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1ca3923586074c5495868f3675094dcb 2024-11-18T03:48:30,286 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/table/1ca3923586074c5495868f3675094dcb is 42, key is table1/table:state/1731901707784/Put/seqid=0 2024-11-18T03:48:30,287 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/table/1ca3923586074c5495868f3675094dcb 2024-11-18T03:48:30,290 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/info/bcc85b146f2f4b7aa392bf6651ac4364 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/info/bcc85b146f2f4b7aa392bf6651ac4364 2024-11-18T03:48:30,291 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/info/bcc85b146f2f4b7aa392bf6651ac4364, delay=999 ms 2024-11-18T03:48:30,293 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/info/bcc85b146f2f4b7aa392bf6651ac4364, entries=10, sequenceid=11, filesize=6.3 K 2024-11-18T03:48:30,294 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/ns/cbb4f7f8e5774d7fbbb95c3ba4016627 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/ns/cbb4f7f8e5774d7fbbb95c3ba4016627 2024-11-18T03:48:30,295 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/ns/cbb4f7f8e5774d7fbbb95c3ba4016627, delay=920 ms 2024-11-18T03:48:30,297 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/ns/cbb4f7f8e5774d7fbbb95c3ba4016627, entries=2, sequenceid=11, filesize=5.0 K 2024-11-18T03:48:30,298 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/.tmp/table/1ca3923586074c5495868f3675094dcb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/table/1ca3923586074c5495868f3675094dcb 2024-11-18T03:48:30,299 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/table/1ca3923586074c5495868f3675094dcb, delay=904 ms 2024-11-18T03:48:30,301 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/table/1ca3923586074c5495868f3675094dcb, entries=2, sequenceid=11, filesize=5.1 K 2024-11-18T03:48:30,302 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false 2024-11-18T03:48:30,302 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T03:48:30,304 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/info/bcc85b146f2f4b7aa392bf6651ac4364 2024-11-18T03:48:30,305 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/ns/cbb4f7f8e5774d7fbbb95c3ba4016627 2024-11-18T03:48:30,305 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/table/1ca3923586074c5495868f3675094dcb 2024-11-18T03:48:30,309 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-18T03:48:30,310 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T03:48:30,310 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:30,310 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731901710208Running coprocessor pre-close hooks at 1731901710208Disabling compacts and flushes for region at 1731901710208Disabling writes for close at 1731901710209 (+1 ms)Obtaining lock to block concurrent updates at 1731901710209Preparing flush snapshotting stores in 1588230740 at 1731901710209Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731901710210 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731901710211 (+1 ms)Flushing 1588230740/info: creating writer at 1731901710211Flushing 1588230740/info: appending metadata at 1731901710236 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731901710236Flushing 1588230740/ns: creating writer at 1731901710243 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731901710264 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731901710264Flushing 1588230740/table: creating writer at 1731901710268 (+4 ms)Flushing 1588230740/table: appending metadata at 1731901710286 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731901710286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@296f4bcb: reopening flushed file at 1731901710290 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43c60451: reopening flushed file at 1731901710294 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@118dfc1e: reopening flushed file at 1731901710297 (+3 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false at 1731901710302 (+5 ms)Writing region close event to WAL at 1731901710306 (+4 ms)Running coprocessor post-close hooks at 1731901710310 (+4 ms)Closed at 1731901710310 2024-11-18T03:48:30,310 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:30,409 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(976): stopping server 88c1dcece5ca,33631,1731901704936; all regions closed. 2024-11-18T03:48:30,414 DEBUG [RS:0;88c1dcece5ca:33631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/oldWALs 2024-11-18T03:48:30,414 INFO [RS:0;88c1dcece5ca:33631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C33631%2C1731901704936.meta:.meta(num 1731901707054) 2024-11-18T03:48:30,416 DEBUG [RS:0;88c1dcece5ca:33631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/oldWALs 2024-11-18T03:48:30,416 INFO [RS:0;88c1dcece5ca:33631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C33631%2C1731901704936:(num 1731901706651) 2024-11-18T03:48:30,416 DEBUG [RS:0;88c1dcece5ca:33631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:30,416 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:30,416 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:30,417 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.ChoreService(370): Chore service for: regionserver/88c1dcece5ca:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:30,417 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:30,417 INFO [regionserver/88c1dcece5ca:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:30,417 INFO [RS:0;88c1dcece5ca:33631 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33631 2024-11-18T03:48:30,423 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:30,423 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88c1dcece5ca,33631,1731901704936 2024-11-18T03:48:30,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:30,426 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88c1dcece5ca,33631,1731901704936] 2024-11-18T03:48:30,428 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88c1dcece5ca,33631,1731901704936 already deleted, retry=false 2024-11-18T03:48:30,428 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 88c1dcece5ca,33631,1731901704936 on 88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:30,432 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 88c1dcece5ca,33631,1731901704936, splitWal=true, meta=true 2024-11-18T03:48:30,435 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 88c1dcece5ca,33631,1731901704936, splitWal=true, meta=true 2024-11-18T03:48:30,435 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 88c1dcece5ca,33631,1731901704936 (carryingMeta=true) 88c1dcece5ca,33631,1731901704936/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@273aaea0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T03:48:30,435 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 88c1dcece5ca,33631,1731901704936, splitWal=true, meta=true, isMeta: true 2024-11-18T03:48:30,436 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936-splitting 2024-11-18T03:48:30,436 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936-splitting dir is empty, no logs to split. 2024-11-18T03:48:30,436 INFO [PEWorker-2 {}] master.SplitWALManager(105): 88c1dcece5ca,33631,1731901704936 WAL count=0, meta=true 2024-11-18T03:48:30,438 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/WALs/88c1dcece5ca,33631,1731901704936-splitting dir is empty, no logs to split. 2024-11-18T03:48:30,438 INFO [PEWorker-2 {}] master.SplitWALManager(105): 88c1dcece5ca,33631,1731901704936 WAL count=0, meta=true 2024-11-18T03:48:30,438 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 88c1dcece5ca,33631,1731901704936 WAL splitting is done? wals=0, meta=true 2024-11-18T03:48:30,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T03:48:30,441 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:48:30,441 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T03:48:30,526 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:30,526 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33631-0x1012930afe20001, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:30,526 INFO [RS:0;88c1dcece5ca:33631 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:30,527 INFO [RS:0;88c1dcece5ca:33631 {}] regionserver.HRegionServer(1031): Exiting; stopping=88c1dcece5ca,33631,1731901704936; zookeeper connection closed. 2024-11-18T03:48:30,527 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77603eca {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77603eca 2024-11-18T03:48:30,592 WARN [88c1dcece5ca:38265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T03:48:31,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T03:48:31,186 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:48:31,186 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:31,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:31,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:31,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:48:31,187 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T03:48:31,187 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2117644383, stopped=false 2024-11-18T03:48:31,188 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=88c1dcece5ca,38265,1731901703942 2024-11-18T03:48:31,188 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '88c1dcece5ca,38265,1731901703942' ***** 2024-11-18T03:48:31,188 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-18T03:48:31,188 INFO [M:0;88c1dcece5ca:38265 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:31,188 INFO [M:0;88c1dcece5ca:38265 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;88c1dcece5ca:38265. 2024-11-18T03:48:31,188 DEBUG [M:0;88c1dcece5ca:38265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:31,188 DEBUG [M:0;88c1dcece5ca:38265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:31,188 INFO [M:0;88c1dcece5ca:38265 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:31,189 DEBUG [M:0;88c1dcece5ca:38265 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T03:48:31,189 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T03:48:31,189 DEBUG [M:0;88c1dcece5ca:38265 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T03:48:31,189 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901706294 {}] cleaner.HFileCleaner(306): Exit Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901706294,5,FailOnTimeoutGroup] 2024-11-18T03:48:31,189 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901706290 {}] cleaner.HFileCleaner(306): Exit Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901706290,5,FailOnTimeoutGroup] 2024-11-18T03:48:31,189 INFO [M:0;88c1dcece5ca:38265 {}] hbase.ChoreService(370): Chore service for: master/88c1dcece5ca:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:31,189 INFO [M:0;88c1dcece5ca:38265 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:31,189 DEBUG [M:0;88c1dcece5ca:38265 {}] master.HMaster(1795): Stopping service threads 2024-11-18T03:48:31,190 INFO [M:0;88c1dcece5ca:38265 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T03:48:31,190 INFO [M:0;88c1dcece5ca:38265 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T03:48:31,190 INFO [M:0;88c1dcece5ca:38265 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T03:48:31,191 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T03:48:31,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:31,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:31,191 DEBUG [M:0;88c1dcece5ca:38265 {}] zookeeper.ZKUtil(347): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T03:48:31,191 WARN [M:0;88c1dcece5ca:38265 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T03:48:31,192 INFO [M:0;88c1dcece5ca:38265 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/.lastflushedseqids 2024-11-18T03:48:31,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:31,193 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T03:48:31,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:31,204 INFO [M:0;88c1dcece5ca:38265 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T03:48:31,342 DEBUG [88c1dcece5ca:38265 {}] assignment.AssignmentManager(2455): Stopped! Dropping assign of 1 queued regions. 2024-11-18T03:48:31,343 INFO [M:0;88c1dcece5ca:38265 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T03:48:31,343 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:48:31,343 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:31,343 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:31,343 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:48:31,343 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:31,343 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.16 KB heapSize=30.77 KB 2024-11-18T03:48:31,354 DEBUG [M:0;88c1dcece5ca:38265 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2caf1757787f4d97a7b2e7facf068db0 2024-11-18T03:48:31,368 DEBUG [M:0;88c1dcece5ca:38265 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2caf1757787f4d97a7b2e7facf068db0 is 82, key is hbase:meta,,1/info:regioninfo/1731901707116/Put/seqid=0 2024-11-18T03:48:31,369 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2caf1757787f4d97a7b2e7facf068db0 2024-11-18T03:48:31,377 DEBUG [M:0;88c1dcece5ca:38265 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 74cc58757a9d46418ba16d4f280349e8 2024-11-18T03:48:31,391 DEBUG [M:0;88c1dcece5ca:38265 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74cc58757a9d46418ba16d4f280349e8 is 705, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731901707789/Put/seqid=0 2024-11-18T03:48:31,392 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.55 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74cc58757a9d46418ba16d4f280349e8 2024-11-18T03:48:31,400 DEBUG [M:0;88c1dcece5ca:38265 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 184a2ecdb73a40a986021e16ea038235 2024-11-18T03:48:31,414 DEBUG [M:0;88c1dcece5ca:38265 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/184a2ecdb73a40a986021e16ea038235 is 58, key is 88c1dcece5ca,33631,1731901704936/rs:/1731901710436/DeleteFamily/seqid=0 2024-11-18T03:48:31,415 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/184a2ecdb73a40a986021e16ea038235 2024-11-18T03:48:31,417 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 184a2ecdb73a40a986021e16ea038235 2024-11-18T03:48:31,418 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2caf1757787f4d97a7b2e7facf068db0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2caf1757787f4d97a7b2e7facf068db0 2024-11-18T03:48:31,419 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2caf1757787f4d97a7b2e7facf068db0, entries=8, sequenceid=64, filesize=5.5 K 2024-11-18T03:48:31,420 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74cc58757a9d46418ba16d4f280349e8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74cc58757a9d46418ba16d4f280349e8 2024-11-18T03:48:31,422 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74cc58757a9d46418ba16d4f280349e8, entries=8, sequenceid=64, filesize=6.5 K 2024-11-18T03:48:31,422 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/184a2ecdb73a40a986021e16ea038235 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/184a2ecdb73a40a986021e16ea038235 2024-11-18T03:48:31,423 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 184a2ecdb73a40a986021e16ea038235 2024-11-18T03:48:31,424 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/737e9941-f85e-cfa7-2035-05e89717287f/ae7edebe-adbf-f94b-90b5-cd3cd35af084/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/184a2ecdb73a40a986021e16ea038235, entries=1, sequenceid=64, filesize=5.2 K 2024-11-18T03:48:31,424 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.16 KB/24735, heapSize ~30.48 KB/31208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=64, compaction requested=false 2024-11-18T03:48:31,427 INFO [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:31,427 DEBUG [M:0;88c1dcece5ca:38265 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731901711343Disabling compacts and flushes for region at 1731901711343Disabling writes for close at 1731901711343Obtaining lock to block concurrent updates at 1731901711343Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731901711343Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24735, getHeapSize=31448, getOffHeapSize=0, getCellsCount=76 at 1731901711344 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731901711344Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731901711344Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731901711368 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731901711368Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731901711371 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731901711390 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731901711390Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731901711394 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731901711414 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731901711414Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ceff86e: reopening flushed file at 1731901711418 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@635205d: reopening flushed file at 1731901711420 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e939fc0: reopening flushed file at 1731901711422 (+2 ms)Finished flush of dataSize ~24.16 KB/24735, heapSize ~30.48 KB/31208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=64, compaction requested=false at 1731901711424 (+2 ms)Writing region close event to WAL at 1731901711426 (+2 ms)Closed at 1731901711427 (+1 ms) 2024-11-18T03:48:31,428 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:31,428 INFO [M:0;88c1dcece5ca:38265 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T03:48:31,429 INFO [M:0;88c1dcece5ca:38265 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38265 2024-11-18T03:48:31,429 INFO [M:0;88c1dcece5ca:38265 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:31,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:31,531 INFO [M:0;88c1dcece5ca:38265 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:31,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38265-0x1012930afe20000, quorum=127.0.0.1:64646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:31,537 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T03:48:31,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T03:48:31,561 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:31,569 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=46 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731901708166 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731901710296 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/88c1dcece5ca:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731901710292 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/88c1dcece5ca:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731901710299 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/88c1dcece5ca:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=574 (was 609), ProcessCount=11 (was 11), AvailableMemoryMB=5818 (was 6638)