2024-11-14 09:23:39,872 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-14 09:23:39,904 main DEBUG Took 0.029916 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 09:23:39,904 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 09:23:39,905 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 09:23:39,905 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 09:23:39,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,913 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 09:23:39,934 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,937 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,939 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,941 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,942 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,943 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,944 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,946 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,947 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,948 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,949 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,950 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:23:39,950 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,951 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 09:23:39,952 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:23:39,954 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 09:23:39,956 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 09:23:39,956 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 09:23:39,958 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 09:23:39,958 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 09:23:39,966 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 09:23:39,969 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 09:23:39,970 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 09:23:39,970 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 09:23:39,971 main DEBUG createAppenders(={Console}) 2024-11-14 09:23:39,971 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-14 09:23:39,972 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-14 09:23:39,972 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-14 09:23:39,972 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 09:23:39,973 main DEBUG OutputStream closed 2024-11-14 09:23:39,973 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 09:23:39,973 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 09:23:39,973 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-14 09:23:40,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 09:23:40,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 09:23:40,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 09:23:40,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 09:23:40,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 09:23:40,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 09:23:40,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 09:23:40,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 09:23:40,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 09:23:40,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 09:23:40,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 09:23:40,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 09:23:40,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 09:23:40,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 09:23:40,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 09:23:40,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 09:23:40,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 09:23:40,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 09:23:40,072 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 09:23:40,073 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-14 09:23:40,073 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 09:23:40,074 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-14T09:23:40,090 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-14 09:23:40,094 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 09:23:40,095 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T09:23:40,380 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b 2024-11-14T09:23:40,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=7111 2024-11-14T09:23:40,521 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T09:23:40,624 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:40,640 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/cluster_d2cca475-7cf7-d931-9b05-18331cca2180, deleteOnExit=true 2024-11-14T09:23:40,734 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/cluster_d2cca475-7cf7-d931-9b05-18331cca2180/zookeeper_0, clientPort=50746, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/cluster_d2cca475-7cf7-d931-9b05-18331cca2180/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/cluster_d2cca475-7cf7-d931-9b05-18331cca2180/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T09:23:40,748 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50746 2024-11-14T09:23:40,749 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:40,846 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4 with version=8 2024-11-14T09:23:40,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/hbase-staging 2024-11-14T09:23:40,946 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T09:23:41,239 INFO [Time-limited test {}] client.ConnectionUtils(128): master/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:23:41,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:23:41,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:23:41,260 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:23:41,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:23:41,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:23:41,417 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T09:23:41,521 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:23:41,567 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T09:23:41,581 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T09:23:41,587 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:23:41,636 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5244 (auto-detected) 2024-11-14T09:23:41,637 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T09:23:41,660 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34451 2024-11-14T09:23:41,682 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34451 connecting to ZooKeeper ensemble=127.0.0.1:50746 2024-11-14T09:23:41,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344510x0, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:23:41,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34451-0x10115ca36b50000 connected 2024-11-14T09:23:41,789 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:41,791 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:41,800 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:23:41,805 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4, hbase.cluster.distributed=false 2024-11-14T09:23:41,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:23:41,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34451 2024-11-14T09:23:41,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34451 2024-11-14T09:23:41,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34451 2024-11-14T09:23:41,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34451 2024-11-14T09:23:41,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34451 2024-11-14T09:23:41,984 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:23:41,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:23:41,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:23:41,988 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:23:41,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:23:41,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:23:41,993 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T09:23:41,997 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:23:41,998 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38601 2024-11-14T09:23:42,001 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38601 connecting to ZooKeeper ensemble=127.0.0.1:50746 2024-11-14T09:23:42,002 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:42,003 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:42,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386010x0, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:23:42,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386010x0, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:23:42,011 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38601-0x10115ca36b50001 connected 2024-11-14T09:23:42,017 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T09:23:42,036 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/bucket.cache 2024-11-14T09:23:42,044 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T09:23:42,046 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T09:23:42,048 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-14T09:23:42,054 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T09:23:42,054 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T09:23:42,061 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T09:23:42,066 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T09:23:42,067 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/bucket.persistence is complete. 2024-11-14T09:23:42,070 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T09:23:42,073 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T09:23:42,080 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:23:42,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38601 2024-11-14T09:23:42,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38601 2024-11-14T09:23:42,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38601 2024-11-14T09:23:42,117 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38601 2024-11-14T09:23:42,119 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38601 2024-11-14T09:23:42,144 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;83f56b55f2af:34451 2024-11-14T09:23:42,146 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/83f56b55f2af,34451,1731576221017 2024-11-14T09:23:42,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:23:42,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:23:42,160 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/83f56b55f2af,34451,1731576221017 2024-11-14T09:23:42,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:42,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T09:23:42,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:42,200 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T09:23:42,204 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/83f56b55f2af,34451,1731576221017 from backup master directory 2024-11-14T09:23:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/83f56b55f2af,34451,1731576221017 2024-11-14T09:23:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:23:42,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:23:42,210 WARN [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:23:42,210 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=83f56b55f2af,34451,1731576221017 2024-11-14T09:23:42,213 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T09:23:42,216 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T09:23:42,241 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/hbase.id] with ID: 651e7055-da4e-4102-8f82-8c72936c582b 2024-11-14T09:23:42,242 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/.tmp/hbase.id 2024-11-14T09:23:42,256 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/hbase.id] 2024-11-14T09:23:42,325 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:42,333 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T09:23:42,362 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-14T09:23:42,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:42,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:42,407 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:23:42,410 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T09:23:42,467 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:23:42,476 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:23:42,574 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store 2024-11-14T09:23:42,600 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T09:23:42,603 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:23:42,604 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:23:42,605 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:42,605 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:42,607 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:23:42,607 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:42,608 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:42,609 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731576222604Disabling compacts and flushes for region at 1731576222604Disabling writes for close at 1731576222607 (+3 ms)Writing region close event to WAL at 1731576222608 (+1 ms)Closed at 1731576222608 2024-11-14T09:23:42,609 WARN [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/.initializing 2024-11-14T09:23:42,609 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/WALs/83f56b55f2af,34451,1731576221017 2024-11-14T09:23:42,623 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:23:42,637 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C34451%2C1731576221017, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/WALs/83f56b55f2af,34451,1731576221017, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/oldWALs, maxLogs=10 2024-11-14T09:23:42,686 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/WALs/83f56b55f2af,34451,1731576221017/83f56b55f2af%2C34451%2C1731576221017.1731576222642 2024-11-14T09:23:42,687 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T09:23:42,688 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:23:42,689 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:23:42,693 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,695 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,786 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T09:23:42,791 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:42,793 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:42,794 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T09:23:42,806 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:42,806 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:23:42,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T09:23:42,815 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:42,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:23:42,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T09:23:42,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:42,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:23:42,830 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,835 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,836 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,841 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,842 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,847 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:23:42,850 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:23:42,863 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:23:42,865 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59162333, jitterRate=-0.11841253936290741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:23:42,874 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731576222714Initializing all the Stores at 1731576222715 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576222715Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576222717 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576222718 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576222718Cleaning up temporary data from old regions at 1731576222843 (+125 ms)Region opened successfully at 1731576222874 (+31 ms) 2024-11-14T09:23:42,885 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T09:23:42,933 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a30fe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:23:42,980 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T09:23:42,992 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T09:23:42,992 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T09:23:42,996 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T09:23:42,996 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T09:23:43,002 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T09:23:43,003 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T09:23:43,032 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T09:23:43,043 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T09:23:43,046 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T09:23:43,050 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T09:23:43,056 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T09:23:43,059 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T09:23:43,062 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T09:23:43,073 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T09:23:43,075 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T09:23:43,077 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T09:23:43,085 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T09:23:43,115 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T09:23:43,118 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T09:23:43,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:23:43,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:23:43,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,137 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=83f56b55f2af,34451,1731576221017, sessionid=0x10115ca36b50000, setting cluster-up flag (Was=false) 2024-11-14T09:23:43,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,165 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T09:23:43,167 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83f56b55f2af,34451,1731576221017 2024-11-14T09:23:43,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,179 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T09:23:43,181 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83f56b55f2af,34451,1731576221017 2024-11-14T09:23:43,190 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T09:23:43,226 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(746): ClusterId : 651e7055-da4e-4102-8f82-8c72936c582b 2024-11-14T09:23:43,230 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T09:23:43,236 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T09:23:43,237 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T09:23:43,242 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T09:23:43,243 DEBUG [RS:0;83f56b55f2af:38601 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5636b5f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:23:43,266 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;83f56b55f2af:38601 2024-11-14T09:23:43,269 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T09:23:43,272 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T09:23:43,272 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T09:23:43,276 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,34451,1731576221017 with port=38601, startcode=1731576221928 2024-11-14T09:23:43,288 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T09:23:43,298 DEBUG [RS:0;83f56b55f2af:38601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T09:23:43,303 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T09:23:43,314 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T09:23:43,323 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 83f56b55f2af,34451,1731576221017 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T09:23:43,333 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:23:43,333 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:23:43,333 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:23:43,334 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:23:43,334 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/83f56b55f2af:0, corePoolSize=10, maxPoolSize=10 2024-11-14T09:23:43,335 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,335 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:23:43,335 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,342 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731576253342 2024-11-14T09:23:43,345 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:23:43,345 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T09:23:43,345 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T09:23:43,346 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T09:23:43,350 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T09:23:43,351 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T09:23:43,351 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:43,351 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T09:23:43,351 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T09:23:43,351 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T09:23:43,360 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,377 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T09:23:43,378 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T09:23:43,379 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T09:23:43,391 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T09:23:43,392 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T09:23:43,396 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731576223393,5,FailOnTimeoutGroup] 2024-11-14T09:23:43,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60099, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T09:23:43,408 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731576223397,5,FailOnTimeoutGroup] 2024-11-14T09:23:43,408 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,408 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T09:23:43,409 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34451 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:23:43,410 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,410 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,415 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T09:23:43,416 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4 2024-11-14T09:23:43,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:23:43,432 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T09:23:43,445 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T09:23:43,445 WARN [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T09:23:43,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:23:43,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:43,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:43,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T09:23:43,454 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:23:43,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:43,455 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:43,455 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T09:23:43,461 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:23:43,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:43,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:43,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T09:23:43,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:23:43,470 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:43,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:43,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:23:43,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740 2024-11-14T09:23:43,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740 2024-11-14T09:23:43,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:23:43,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:23:43,474 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:23:43,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:23:43,487 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:23:43,488 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69002574, jitterRate=0.028218477964401245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:23:43,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731576223431Initializing all the Stores at 1731576223431Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576223431Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576223432 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576223432Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576223432Cleaning up temporary data from old regions at 1731576223474 (+42 ms)Region opened successfully at 1731576223491 (+17 ms) 2024-11-14T09:23:43,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:23:43,492 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:23:43,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:23:43,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:23:43,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:23:43,494 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:23:43,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731576223492Disabling compacts and flushes for region at 1731576223492Disabling writes for close at 1731576223492Writing region close event to WAL at 1731576223493 (+1 ms)Closed at 1731576223494 (+1 ms) 2024-11-14T09:23:43,497 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:23:43,497 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T09:23:43,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T09:23:43,514 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:23:43,517 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T09:23:43,547 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,34451,1731576221017 with port=38601, startcode=1731576221928 2024-11-14T09:23:43,550 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34451 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 83f56b55f2af,38601,1731576221928 2024-11-14T09:23:43,554 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34451 {}] master.ServerManager(517): Registering regionserver=83f56b55f2af,38601,1731576221928 2024-11-14T09:23:43,564 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4 2024-11-14T09:23:43,564 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T09:23:43,564 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T09:23:43,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:23:43,570 DEBUG [RS:0;83f56b55f2af:38601 {}] zookeeper.ZKUtil(111): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/83f56b55f2af,38601,1731576221928 2024-11-14T09:23:43,570 WARN [RS:0;83f56b55f2af:38601 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:23:43,570 INFO [RS:0;83f56b55f2af:38601 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:23:43,571 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928 2024-11-14T09:23:43,573 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [83f56b55f2af,38601,1731576221928] 2024-11-14T09:23:43,597 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T09:23:43,612 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T09:23:43,617 INFO [RS:0;83f56b55f2af:38601 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:23:43,617 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,618 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T09:23:43,623 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T09:23:43,624 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,625 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,625 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,625 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,625 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,625 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,625 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:23:43,625 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,626 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,626 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,626 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,626 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,626 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:23:43,626 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:23:43,627 DEBUG [RS:0;83f56b55f2af:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:23:43,630 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,630 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,630 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,630 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,631 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,631 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,38601,1731576221928-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:23:43,655 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T09:23:43,658 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,38601,1731576221928-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,659 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,659 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.Replication(171): 83f56b55f2af,38601,1731576221928 started 2024-11-14T09:23:43,668 WARN [83f56b55f2af:34451 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T09:23:43,685 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:43,686 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1482): Serving as 83f56b55f2af,38601,1731576221928, RpcServer on 83f56b55f2af/172.17.0.2:38601, sessionid=0x10115ca36b50001 2024-11-14T09:23:43,687 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T09:23:43,687 DEBUG [RS:0;83f56b55f2af:38601 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 83f56b55f2af,38601,1731576221928 2024-11-14T09:23:43,687 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,38601,1731576221928' 2024-11-14T09:23:43,687 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T09:23:43,688 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T09:23:43,689 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T09:23:43,689 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T09:23:43,689 DEBUG [RS:0;83f56b55f2af:38601 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 83f56b55f2af,38601,1731576221928 2024-11-14T09:23:43,689 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,38601,1731576221928' 2024-11-14T09:23:43,689 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T09:23:43,690 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T09:23:43,691 DEBUG [RS:0;83f56b55f2af:38601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T09:23:43,691 INFO [RS:0;83f56b55f2af:38601 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T09:23:43,691 INFO [RS:0;83f56b55f2af:38601 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T09:23:43,799 INFO [RS:0;83f56b55f2af:38601 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:23:43,800 INFO [RS:0;83f56b55f2af:38601 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=83f56b55f2af%2C38601%2C1731576221928, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/oldWALs, maxLogs=55 2024-11-14T09:23:43,839 INFO [RS:0;83f56b55f2af:38601 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928/83f56b55f2af%2C38601%2C1731576221928.1731576223804 2024-11-14T09:23:43,840 DEBUG [RS:0;83f56b55f2af:38601 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T09:23:43,922 DEBUG [83f56b55f2af:34451 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T09:23:43,939 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=83f56b55f2af,38601,1731576221928 2024-11-14T09:23:43,947 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,38601,1731576221928, state=OPENING 2024-11-14T09:23:43,955 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T09:23:43,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:43,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:23:43,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:23:43,960 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:23:43,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,38601,1731576221928}] 2024-11-14T09:23:44,145 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:23:44,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57801, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:23:44,159 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T09:23:44,160 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:23:44,161 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T09:23:44,162 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=83f56b55f2af%2C38601%2C1731576221928.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/oldWALs, maxLogs=55 2024-11-14T09:23:44,191 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928/83f56b55f2af%2C38601%2C1731576221928.meta.1731576224164.meta 2024-11-14T09:23:44,192 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T09:23:44,192 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:23:44,195 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T09:23:44,198 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T09:23:44,204 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T09:23:44,210 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T09:23:44,210 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:23:44,211 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T09:23:44,211 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T09:23:44,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T09:23:44,215 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:23:44,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:44,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:44,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T09:23:44,217 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:23:44,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:44,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:44,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T09:23:44,219 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:23:44,219 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:44,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:44,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T09:23:44,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:23:44,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:44,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:23:44,221 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:23:44,223 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740 2024-11-14T09:23:44,224 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740 2024-11-14T09:23:44,225 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:23:44,225 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:23:44,226 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:23:44,227 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:23:44,229 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75219842, jitterRate=0.12086299061775208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:23:44,229 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T09:23:44,231 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731576224212Writing region info on filesystem at 1731576224212Initializing all the Stores at 1731576224212Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576224213 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576224213Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576224213Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576224213Cleaning up temporary data from old regions at 1731576224225 (+12 ms)Running coprocessor post-open hooks at 1731576224229 (+4 ms)Region opened successfully at 1731576224230 (+1 ms) 2024-11-14T09:23:44,237 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731576224133 2024-11-14T09:23:44,250 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T09:23:44,250 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T09:23:44,251 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,38601,1731576221928 2024-11-14T09:23:44,253 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,38601,1731576221928, state=OPEN 2024-11-14T09:23:44,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:23:44,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:23:44,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:23:44,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:23:44,260 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=83f56b55f2af,38601,1731576221928 2024-11-14T09:23:44,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T09:23:44,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,38601,1731576221928 in 297 msec 2024-11-14T09:23:44,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T09:23:44,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 762 msec 2024-11-14T09:23:44,272 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:23:44,272 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T09:23:44,302 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:23:44,303 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,38601,1731576221928, seqNum=-1] 2024-11-14T09:23:44,330 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:23:44,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44391, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:23:44,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1360 sec 2024-11-14T09:23:44,362 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731576224362, completionTime=-1 2024-11-14T09:23:44,365 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T09:23:44,365 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T09:23:44,392 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T09:23:44,392 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731576284392 2024-11-14T09:23:44,392 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731576344392 2024-11-14T09:23:44,392 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-14T09:23:44,396 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,34451,1731576221017-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:44,396 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,34451,1731576221017-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:44,396 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,34451,1731576221017-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:44,398 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-83f56b55f2af:34451, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:44,398 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:44,399 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:44,404 DEBUG [master/83f56b55f2af:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T09:23:44,425 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.215sec 2024-11-14T09:23:44,427 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T09:23:44,428 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T09:23:44,429 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T09:23:44,429 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T09:23:44,429 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T09:23:44,430 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,34451,1731576221017-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:23:44,431 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,34451,1731576221017-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T09:23:44,435 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T09:23:44,436 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T09:23:44,436 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,34451,1731576221017-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:23:44,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567b1492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:23:44,537 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T09:23:44,537 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T09:23:44,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,34451,-1 for getting cluster id 2024-11-14T09:23:44,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:23:44,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e7055-da4e-4102-8f82-8c72936c582b' 2024-11-14T09:23:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:23:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e7055-da4e-4102-8f82-8c72936c582b" 2024-11-14T09:23:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f54d6f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:23:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,34451,-1] 2024-11-14T09:23:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:23:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:23:44,581 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:23:44,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3d5ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:23:44,588 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:23:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,38601,1731576221928, seqNum=-1] 2024-11-14T09:23:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:23:44,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:23:44,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=83f56b55f2af,34451,1731576221017 2024-11-14T09:23:44,631 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:23:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,34451,1731576221017 2024-11-14T09:23:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e887806 2024-11-14T09:23:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:23:44,647 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:23:44,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34451 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:23:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34451 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-14T09:23:44,665 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:23:44,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34451 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-14T09:23:44,668 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:44,674 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:23:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:23:44,694 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f4477aebba38cf08f468eadf035de823, NAME => 'table1,,1731576224649.f4477aebba38cf08f468eadf035de823.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4 2024-11-14T09:23:44,704 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731576224649.f4477aebba38cf08f468eadf035de823.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:23:44,704 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing f4477aebba38cf08f468eadf035de823, disabling compactions & flushes 2024-11-14T09:23:44,704 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:44,704 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:44,705 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731576224649.f4477aebba38cf08f468eadf035de823. after waiting 0 ms 2024-11-14T09:23:44,705 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:44,705 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:44,705 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for f4477aebba38cf08f468eadf035de823: Waiting for close lock at 1731576224704Disabling compacts and flushes for region at 1731576224704Disabling writes for close at 1731576224705 (+1 ms)Writing region close event to WAL at 1731576224705Closed at 1731576224705 2024-11-14T09:23:44,707 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:23:44,718 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731576224649.f4477aebba38cf08f468eadf035de823.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731576224707"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731576224707"}]},"ts":"1731576224707"} 2024-11-14T09:23:44,723 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:23:44,725 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:23:44,729 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731576224725"}]},"ts":"1731576224725"} 2024-11-14T09:23:44,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-14T09:23:44,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=f4477aebba38cf08f468eadf035de823, ASSIGN}] 2024-11-14T09:23:44,738 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=f4477aebba38cf08f468eadf035de823, ASSIGN 2024-11-14T09:23:44,740 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=f4477aebba38cf08f468eadf035de823, ASSIGN; state=OFFLINE, location=83f56b55f2af,38601,1731576221928; forceNewPlan=false, retain=false 2024-11-14T09:23:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:23:44,891 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f4477aebba38cf08f468eadf035de823, regionState=OPENING, regionLocation=83f56b55f2af,38601,1731576221928 2024-11-14T09:23:44,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=f4477aebba38cf08f468eadf035de823, ASSIGN because future has completed 2024-11-14T09:23:44,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4477aebba38cf08f468eadf035de823, server=83f56b55f2af,38601,1731576221928}] 2024-11-14T09:23:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:23:45,054 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:45,054 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f4477aebba38cf08f468eadf035de823, NAME => 'table1,,1731576224649.f4477aebba38cf08f468eadf035de823.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:23:45,055 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,055 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731576224649.f4477aebba38cf08f468eadf035de823.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:23:45,055 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,055 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,058 INFO [StoreOpener-f4477aebba38cf08f468eadf035de823-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,065 INFO [StoreOpener-f4477aebba38cf08f468eadf035de823-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4477aebba38cf08f468eadf035de823 columnFamilyName family 2024-11-14T09:23:45,065 DEBUG [StoreOpener-f4477aebba38cf08f468eadf035de823-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:23:45,066 INFO [StoreOpener-f4477aebba38cf08f468eadf035de823-1 {}] regionserver.HStore(327): Store=f4477aebba38cf08f468eadf035de823/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:23:45,066 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,067 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,068 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,068 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,068 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,069 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,081 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:23:45,081 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f4477aebba38cf08f468eadf035de823; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58758750, jitterRate=-0.12442639470100403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:23:45,081 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:45,082 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f4477aebba38cf08f468eadf035de823: Running coprocessor pre-open hook at 1731576225056Writing region info on filesystem at 1731576225056Initializing all the Stores at 1731576225056Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576225056Cleaning up temporary data from old regions at 1731576225068 (+12 ms)Running coprocessor post-open hooks at 1731576225081 (+13 ms)Region opened successfully at 1731576225082 (+1 ms) 2024-11-14T09:23:45,083 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731576224649.f4477aebba38cf08f468eadf035de823., pid=6, masterSystemTime=1731576225050 2024-11-14T09:23:45,087 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:45,087 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:45,087 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f4477aebba38cf08f468eadf035de823, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,38601,1731576221928 2024-11-14T09:23:45,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4477aebba38cf08f468eadf035de823, server=83f56b55f2af,38601,1731576221928 because future has completed 2024-11-14T09:23:45,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T09:23:45,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f4477aebba38cf08f468eadf035de823, server=83f56b55f2af,38601,1731576221928 in 195 msec 2024-11-14T09:23:45,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T09:23:45,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=f4477aebba38cf08f468eadf035de823, ASSIGN in 358 msec 2024-11-14T09:23:45,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:23:45,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731576225097"}]},"ts":"1731576225097"} 2024-11-14T09:23:45,099 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-14T09:23:45,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:23:45,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 443 msec 2024-11-14T09:23:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:23:45,304 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-14T09:23:45,304 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-14T09:23:45,307 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T09:23:45,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-14T09:23:45,315 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T09:23:45,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-14T09:23:45,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731576224649.f4477aebba38cf08f468eadf035de823., hostname=83f56b55f2af,38601,1731576221928, seqNum=2] 2024-11-14T09:23:45,339 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f4477aebba38cf08f468eadf035de823 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-14T09:23:45,367 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3e2f8d2ed42345b09bdb0dcc3ca54499 2024-11-14T09:23:45,404 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/.tmp/family/3e2f8d2ed42345b09bdb0dcc3ca54499 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-14T09:23:45,409 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/.tmp/family/3e2f8d2ed42345b09bdb0dcc3ca54499 2024-11-14T09:23:45,449 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/.tmp/family/3e2f8d2ed42345b09bdb0dcc3ca54499 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499 2024-11-14T09:23:45,451 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499, delay=941 ms 2024-11-14T09:23:45,453 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499, entries=2, sequenceid=6, filesize=4.9 K 2024-11-14T09:23:45,460 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f4477aebba38cf08f468eadf035de823 in 120ms, sequenceid=6, compaction requested=false 2024-11-14T09:23:45,461 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-14T09:23:45,463 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f4477aebba38cf08f468eadf035de823: 2024-11-14T09:23:46,394 DEBUG [hfile-prefetch-1731576225451 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3e2f8d2ed42345b09bdb0dcc3ca54499_0. 2024-11-14T09:23:46,398 DEBUG [hfile-prefetch-1731576225451 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3e2f8d2ed42345b09bdb0dcc3ca54499_117. 2024-11-14T09:23:46,399 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499, with total blocks 2, and data blocks 1 2024-11-14T09:23:46,399 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T09:23:46,400 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T09:23:46,400 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499. Will sleep 100ms and try the verification again. 2024-11-14T09:23:46,500 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499, with total blocks 2, and data blocks 1 2024-11-14T09:23:46,500 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-14T09:23:46,500 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2294): found block for file 3e2f8d2ed42345b09bdb0dcc3ca54499 in the backing map. Acquiring read lock for offset 0 2024-11-14T09:23:46,501 DEBUG [hfile-prefetch-1731576225451 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/family/3e2f8d2ed42345b09bdb0dcc3ca54499 has now been fully cached. 2024-11-14T09:23:46,501 DEBUG [hfile-prefetch-1731576225451 {}] hfile.PrefetchExecutor(121): Prefetch completed for 3e2f8d2ed42345b09bdb0dcc3ca54499 2024-11-14T09:23:47,053 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T09:23:47,053 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T09:23:47,054 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T09:23:47,054 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-14T09:23:47,080 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-14T09:23:47,080 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731576227053 2024-11-14T09:23:47,080 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T09:23:47,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:23:47,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:23:47,474 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;83f56b55f2af:38601,5,FailOnTimeoutGroup] 2024-11-14T09:23:47,474 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '83f56b55f2af,38601,1731576221928' ***** 2024-11-14T09:23:47,474 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-14T09:23:47,474 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-14T09:23:47,475 INFO [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T09:23:47,475 INFO [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/bucket.persistence 2024-11-14T09:23:47,475 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T09:23:47,475 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T09:23:47,475 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T09:23:47,475 INFO [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T09:23:47,475 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T09:23:47,486 DEBUG [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1488): Thread RS:0;83f56b55f2af:38601 started persisting bucket cache to file 2024-11-14T09:23:47,486 DEBUG [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T09:23:47,486 DEBUG [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-14T09:23:47,489 DEBUG [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-14T09:23:47,489 DEBUG [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731576227486 2024-11-14T09:23:47,489 DEBUG [RS:0;83f56b55f2af:38601 {}] bucket.BucketCache(1510): Thread RS:0;83f56b55f2af:38601 finished persisting bucket cache to file, renaming 2024-11-14T09:23:47,489 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T09:23:47,490 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T09:23:47,490 INFO [RS:0;83f56b55f2af:38601 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T09:23:47,490 INFO [RS:0;83f56b55f2af:38601 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T09:23:47,490 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(3091): Received CLOSE for f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:47,491 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(959): stopping server 83f56b55f2af,38601,1731576221928 2024-11-14T09:23:47,491 INFO [RS:0;83f56b55f2af:38601 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:23:47,491 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731576224649.f4477aebba38cf08f468eadf035de823. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T09:23:47,491 INFO [RS:0;83f56b55f2af:38601 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;83f56b55f2af:38601. 2024-11-14T09:23:47,491 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f4477aebba38cf08f468eadf035de823, disabling compactions & flushes 2024-11-14T09:23:47,491 DEBUG [RS:0;83f56b55f2af:38601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:23:47,491 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:47,492 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:47,492 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731576224649.f4477aebba38cf08f468eadf035de823. after waiting 0 ms 2024-11-14T09:23:47,492 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:47,498 DEBUG [RS:0;83f56b55f2af:38601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:23:47,498 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T09:23:47,498 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T09:23:47,499 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T09:23:47,499 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T09:23:47,499 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T09:23:47,499 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1325): Online Regions={f4477aebba38cf08f468eadf035de823=table1,,1731576224649.f4477aebba38cf08f468eadf035de823., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T09:23:47,499 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T09:23:47,500 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:23:47,500 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:23:47,500 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:23:47,500 DEBUG [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f4477aebba38cf08f468eadf035de823 2024-11-14T09:23:47,500 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:23:47,500 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:23:47,500 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-14T09:23:47,506 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/default/table1/f4477aebba38cf08f468eadf035de823/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T09:23:47,507 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:47,508 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f4477aebba38cf08f468eadf035de823: Waiting for close lock at 1731576227491Running coprocessor pre-close hooks at 1731576227491Disabling compacts and flushes for region at 1731576227491Disabling writes for close at 1731576227492 (+1 ms)Writing region close event to WAL at 1731576227496 (+4 ms)Running coprocessor post-close hooks at 1731576227506 (+10 ms)Closed at 1731576227507 (+1 ms) 2024-11-14T09:23:47,508 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731576224649.f4477aebba38cf08f468eadf035de823. 2024-11-14T09:23:47,514 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6637a2d89c3e41cd8f5c9e2f10d98bba 2024-11-14T09:23:47,531 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/info/6637a2d89c3e41cd8f5c9e2f10d98bba is 133, key is table1,,1731576224649.f4477aebba38cf08f468eadf035de823./info:regioninfo/1731576225087/Put/seqid=0 2024-11-14T09:23:47,532 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/info/6637a2d89c3e41cd8f5c9e2f10d98bba 2024-11-14T09:23:47,534 INFO [regionserver/83f56b55f2af:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:23:47,547 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d999981253e54155a60e99aabc973071 2024-11-14T09:23:47,561 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/ns/d999981253e54155a60e99aabc973071 is 43, key is default/ns:d/1731576224339/Put/seqid=0 2024-11-14T09:23:47,562 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/ns/d999981253e54155a60e99aabc973071 2024-11-14T09:23:47,573 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 35b924cc022d4e0d83d3ebcca7029783 2024-11-14T09:23:47,589 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/table/35b924cc022d4e0d83d3ebcca7029783 is 42, key is table1/table:state/1731576225097/Put/seqid=0 2024-11-14T09:23:47,590 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/table/35b924cc022d4e0d83d3ebcca7029783 2024-11-14T09:23:47,594 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/info/6637a2d89c3e41cd8f5c9e2f10d98bba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/info/6637a2d89c3e41cd8f5c9e2f10d98bba 2024-11-14T09:23:47,595 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/info/6637a2d89c3e41cd8f5c9e2f10d98bba, delay=938 ms 2024-11-14T09:23:47,597 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/info/6637a2d89c3e41cd8f5c9e2f10d98bba, entries=10, sequenceid=11, filesize=6.3 K 2024-11-14T09:23:47,598 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/ns/d999981253e54155a60e99aabc973071 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/ns/d999981253e54155a60e99aabc973071 2024-11-14T09:23:47,600 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/ns/d999981253e54155a60e99aabc973071, delay=977 ms 2024-11-14T09:23:47,602 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/ns/d999981253e54155a60e99aabc973071, entries=2, sequenceid=11, filesize=5.0 K 2024-11-14T09:23:47,602 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/.tmp/table/35b924cc022d4e0d83d3ebcca7029783 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/table/35b924cc022d4e0d83d3ebcca7029783 2024-11-14T09:23:47,604 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/table/35b924cc022d4e0d83d3ebcca7029783, delay=926 ms 2024-11-14T09:23:47,605 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/table/35b924cc022d4e0d83d3ebcca7029783, entries=2, sequenceid=11, filesize=5.1 K 2024-11-14T09:23:47,606 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false 2024-11-14T09:23:47,606 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T09:23:47,608 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/info/6637a2d89c3e41cd8f5c9e2f10d98bba 2024-11-14T09:23:47,609 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/ns/d999981253e54155a60e99aabc973071 2024-11-14T09:23:47,610 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/table/35b924cc022d4e0d83d3ebcca7029783 2024-11-14T09:23:47,615 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-14T09:23:47,616 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:23:47,616 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:23:47,616 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731576227499Running coprocessor pre-close hooks at 1731576227499Disabling compacts and flushes for region at 1731576227499Disabling writes for close at 1731576227500 (+1 ms)Obtaining lock to block concurrent updates at 1731576227500Preparing flush snapshotting stores in 1588230740 at 1731576227500Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731576227501 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731576227501Flushing 1588230740/info: creating writer at 1731576227502 (+1 ms)Flushing 1588230740/info: appending metadata at 1731576227530 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731576227530Flushing 1588230740/ns: creating writer at 1731576227535 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731576227559 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1731576227559Flushing 1588230740/table: creating writer at 1731576227566 (+7 ms)Flushing 1588230740/table: appending metadata at 1731576227588 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731576227588Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b14de2b: reopening flushed file at 1731576227593 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24ad427a: reopening flushed file at 1731576227598 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@186b5934: reopening flushed file at 1731576227602 (+4 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false at 1731576227606 (+4 ms)Writing region close event to WAL at 1731576227611 (+5 ms)Running coprocessor post-close hooks at 1731576227616 (+5 ms)Closed at 1731576227616 2024-11-14T09:23:47,617 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T09:23:47,636 INFO [regionserver/83f56b55f2af:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T09:23:47,636 INFO [regionserver/83f56b55f2af:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T09:23:47,700 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(976): stopping server 83f56b55f2af,38601,1731576221928; all regions closed. 2024-11-14T09:23:47,706 DEBUG [RS:0;83f56b55f2af:38601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/oldWALs 2024-11-14T09:23:47,706 INFO [RS:0;83f56b55f2af:38601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83f56b55f2af%2C38601%2C1731576221928.meta:.meta(num 1731576224164) 2024-11-14T09:23:47,708 DEBUG [RS:0;83f56b55f2af:38601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/oldWALs 2024-11-14T09:23:47,709 INFO [RS:0;83f56b55f2af:38601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83f56b55f2af%2C38601%2C1731576221928:(num 1731576223804) 2024-11-14T09:23:47,709 DEBUG [RS:0;83f56b55f2af:38601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:23:47,709 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:23:47,709 INFO [RS:0;83f56b55f2af:38601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:23:47,709 INFO [RS:0;83f56b55f2af:38601 {}] hbase.ChoreService(370): Chore service for: regionserver/83f56b55f2af:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T09:23:47,710 INFO [RS:0;83f56b55f2af:38601 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:23:47,710 INFO [regionserver/83f56b55f2af:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:23:47,710 INFO [RS:0;83f56b55f2af:38601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38601 2024-11-14T09:23:47,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:23:47,716 INFO [RS:0;83f56b55f2af:38601 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:23:47,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/83f56b55f2af,38601,1731576221928 2024-11-14T09:23:47,717 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [83f56b55f2af,38601,1731576221928] 2024-11-14T09:23:47,720 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/83f56b55f2af,38601,1731576221928 already deleted, retry=false 2024-11-14T09:23:47,720 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 83f56b55f2af,38601,1731576221928 on 83f56b55f2af,34451,1731576221017 2024-11-14T09:23:47,725 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 83f56b55f2af,38601,1731576221928, splitWal=true, meta=true 2024-11-14T09:23:47,727 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 83f56b55f2af,38601,1731576221928, splitWal=true, meta=true 2024-11-14T09:23:47,727 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for 83f56b55f2af,38601,1731576221928 (carryingMeta=true) 83f56b55f2af,38601,1731576221928/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@114eb0d1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T09:23:47,729 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 83f56b55f2af,38601,1731576221928, splitWal=true, meta=true, isMeta: true 2024-11-14T09:23:47,729 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928-splitting 2024-11-14T09:23:47,730 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928-splitting dir is empty, no logs to split. 2024-11-14T09:23:47,730 INFO [PEWorker-2 {}] master.SplitWALManager(105): 83f56b55f2af,38601,1731576221928 WAL count=0, meta=true 2024-11-14T09:23:47,732 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/WALs/83f56b55f2af,38601,1731576221928-splitting dir is empty, no logs to split. 2024-11-14T09:23:47,732 INFO [PEWorker-2 {}] master.SplitWALManager(105): 83f56b55f2af,38601,1731576221928 WAL count=0, meta=true 2024-11-14T09:23:47,732 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 83f56b55f2af,38601,1731576221928 WAL splitting is done? wals=0, meta=true 2024-11-14T09:23:47,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T09:23:47,735 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:23:47,735 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T09:23:47,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:23:47,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x10115ca36b50001, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:23:47,820 INFO [RS:0;83f56b55f2af:38601 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:23:47,820 INFO [RS:0;83f56b55f2af:38601 {}] regionserver.HRegionServer(1031): Exiting; stopping=83f56b55f2af,38601,1731576221928; zookeeper connection closed. 2024-11-14T09:23:47,821 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7ff7ab6f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7ff7ab6f 2024-11-14T09:23:47,886 WARN [83f56b55f2af:34451 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T09:23:48,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T09:23:48,475 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:23:48,475 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:23:48,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:23:48,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:23:48,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:23:48,476 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T09:23:48,476 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1590087585, stopped=false 2024-11-14T09:23:48,477 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=83f56b55f2af,34451,1731576221017 2024-11-14T09:23:48,477 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '83f56b55f2af,34451,1731576221017' ***** 2024-11-14T09:23:48,477 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-14T09:23:48,477 INFO [M:0;83f56b55f2af:34451 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:23:48,477 INFO [M:0;83f56b55f2af:34451 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;83f56b55f2af:34451. 2024-11-14T09:23:48,477 DEBUG [M:0;83f56b55f2af:34451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:23:48,477 DEBUG [M:0;83f56b55f2af:34451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:23:48,477 INFO [M:0;83f56b55f2af:34451 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:23:48,478 DEBUG [M:0;83f56b55f2af:34451 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T09:23:48,478 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T09:23:48,478 DEBUG [M:0;83f56b55f2af:34451 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T09:23:48,478 DEBUG [master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731576223393 {}] cleaner.HFileCleaner(306): Exit Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731576223393,5,FailOnTimeoutGroup] 2024-11-14T09:23:48,478 DEBUG [master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731576223397 {}] cleaner.HFileCleaner(306): Exit Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731576223397,5,FailOnTimeoutGroup] 2024-11-14T09:23:48,478 INFO [M:0;83f56b55f2af:34451 {}] hbase.ChoreService(370): Chore service for: master/83f56b55f2af:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T09:23:48,478 INFO [M:0;83f56b55f2af:34451 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:23:48,478 DEBUG [M:0;83f56b55f2af:34451 {}] master.HMaster(1795): Stopping service threads 2024-11-14T09:23:48,478 INFO [M:0;83f56b55f2af:34451 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T09:23:48,479 INFO [M:0;83f56b55f2af:34451 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:23:48,480 INFO [M:0;83f56b55f2af:34451 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T09:23:48,480 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T09:23:48,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T09:23:48,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:23:48,481 DEBUG [M:0;83f56b55f2af:34451 {}] zookeeper.ZKUtil(347): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T09:23:48,481 WARN [M:0;83f56b55f2af:34451 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T09:23:48,482 INFO [M:0;83f56b55f2af:34451 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/.lastflushedseqids 2024-11-14T09:23:48,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:23:48,483 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T09:23:48,484 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:23:48,494 INFO [M:0;83f56b55f2af:34451 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T09:23:48,636 DEBUG [83f56b55f2af:34451 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-14T09:23:48,636 INFO [M:0;83f56b55f2af:34451 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T09:23:48,637 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:23:48,637 INFO [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:48,637 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:48,637 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:23:48,637 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:48,637 INFO [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-11-14T09:23:48,647 DEBUG [M:0;83f56b55f2af:34451 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b655dc0d08574e6cb27325a2eb13c51b 2024-11-14T09:23:48,661 DEBUG [M:0;83f56b55f2af:34451 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b655dc0d08574e6cb27325a2eb13c51b is 82, key is hbase:meta,,1/info:regioninfo/1731576224250/Put/seqid=0 2024-11-14T09:23:48,662 INFO [M:0;83f56b55f2af:34451 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b655dc0d08574e6cb27325a2eb13c51b 2024-11-14T09:23:48,670 DEBUG [M:0;83f56b55f2af:34451 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ae797b2f8d9043dfabb8b791a3286b66 2024-11-14T09:23:48,683 DEBUG [M:0;83f56b55f2af:34451 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae797b2f8d9043dfabb8b791a3286b66 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731576225101/Put/seqid=0 2024-11-14T09:23:48,684 INFO [M:0;83f56b55f2af:34451 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae797b2f8d9043dfabb8b791a3286b66 2024-11-14T09:23:48,692 DEBUG [M:0;83f56b55f2af:34451 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b41dd8fb905b4e9dae753388bd12e303 2024-11-14T09:23:48,704 DEBUG [M:0;83f56b55f2af:34451 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b41dd8fb905b4e9dae753388bd12e303 is 58, key is 83f56b55f2af,38601,1731576221928/rs:/1731576227728/DeleteFamily/seqid=0 2024-11-14T09:23:48,705 INFO [M:0;83f56b55f2af:34451 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b41dd8fb905b4e9dae753388bd12e303 2024-11-14T09:23:48,707 INFO [M:0;83f56b55f2af:34451 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b41dd8fb905b4e9dae753388bd12e303 2024-11-14T09:23:48,707 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b655dc0d08574e6cb27325a2eb13c51b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b655dc0d08574e6cb27325a2eb13c51b 2024-11-14T09:23:48,709 INFO [M:0;83f56b55f2af:34451 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b655dc0d08574e6cb27325a2eb13c51b, entries=8, sequenceid=64, filesize=5.5 K 2024-11-14T09:23:48,709 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae797b2f8d9043dfabb8b791a3286b66 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ae797b2f8d9043dfabb8b791a3286b66 2024-11-14T09:23:48,710 INFO [M:0;83f56b55f2af:34451 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ae797b2f8d9043dfabb8b791a3286b66, entries=8, sequenceid=64, filesize=6.5 K 2024-11-14T09:23:48,711 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b41dd8fb905b4e9dae753388bd12e303 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b41dd8fb905b4e9dae753388bd12e303 2024-11-14T09:23:48,712 INFO [M:0;83f56b55f2af:34451 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b41dd8fb905b4e9dae753388bd12e303 2024-11-14T09:23:48,713 INFO [M:0;83f56b55f2af:34451 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f17baea2-908d-c620-afcd-350ef423131b/11209fed-8478-bf48-b94a-71ab30d3d9b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b41dd8fb905b4e9dae753388bd12e303, entries=1, sequenceid=64, filesize=5.2 K 2024-11-14T09:23:48,713 INFO [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=64, compaction requested=false 2024-11-14T09:23:48,715 INFO [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:23:48,715 DEBUG [M:0;83f56b55f2af:34451 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731576228637Disabling compacts and flushes for region at 1731576228637Disabling writes for close at 1731576228637Obtaining lock to block concurrent updates at 1731576228637Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731576228637Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731576228638 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731576228638Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731576228638Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731576228660 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731576228660Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731576228663 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731576228683 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731576228683Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731576228686 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731576228703 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731576228703Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18b1b7c2: reopening flushed file at 1731576228707 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@438ac082: reopening flushed file at 1731576228709 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a06bb1: reopening flushed file at 1731576228711 (+2 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=64, compaction requested=false at 1731576228713 (+2 ms)Writing region close event to WAL at 1731576228715 (+2 ms)Closed at 1731576228715 2024-11-14T09:23:48,717 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:23:48,717 INFO [M:0;83f56b55f2af:34451 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T09:23:48,717 INFO [M:0;83f56b55f2af:34451 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34451 2024-11-14T09:23:48,718 INFO [M:0;83f56b55f2af:34451 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:23:48,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:23:48,820 INFO [M:0;83f56b55f2af:34451 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:23:48,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34451-0x10115ca36b50000, quorum=127.0.0.1:50746, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:23:48,824 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T09:23:48,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T09:23:48,846 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:23:48,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731576225451 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731576227604 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731576227595 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/83f56b55f2af:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731576227600 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/83f56b55f2af:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/83f56b55f2af:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=270 (was 294), ProcessCount=11 (was 11), AvailableMemoryMB=7750 (was 7111) - AvailableMemoryMB LEAK? -