2024-12-08 05:45:25,125 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-12-08 05:45:25,144 main DEBUG Took 0.016716 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 05:45:25,145 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 05:45:25,145 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 05:45:25,146 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 05:45:25,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,156 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 05:45:25,171 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,173 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,174 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,176 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,177 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,179 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,180 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,181 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,182 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,183 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,184 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,184 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,185 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,185 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 05:45:25,186 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,186 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 05:45:25,188 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 05:45:25,189 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 05:45:25,191 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 05:45:25,192 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 05:45:25,193 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 05:45:25,194 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 05:45:25,205 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 05:45:25,208 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 05:45:25,210 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 05:45:25,211 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 05:45:25,211 main DEBUG createAppenders(={Console}) 2024-12-08 05:45:25,212 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-12-08 05:45:25,212 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-12-08 05:45:25,213 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-12-08 05:45:25,213 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 05:45:25,214 main DEBUG OutputStream closed 2024-12-08 05:45:25,214 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 05:45:25,215 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 05:45:25,215 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-12-08 05:45:25,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 05:45:25,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 05:45:25,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 05:45:25,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 05:45:25,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 05:45:25,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 05:45:25,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 05:45:25,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 05:45:25,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 05:45:25,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 05:45:25,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 05:45:25,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 05:45:25,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 05:45:25,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 05:45:25,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 05:45:25,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 05:45:25,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 05:45:25,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 05:45:25,310 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 05:45:25,310 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-12-08 05:45:25,311 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 05:45:25,311 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-12-08T05:45:25,325 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-12-08 05:45:25,328 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 05:45:25,329 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T05:45:25,576 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976 2024-12-08T05:45:25,603 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=8463 2024-12-08T05:45:25,696 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T05:45:25,790 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:25,805 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/cluster_8140fb43-a50a-3469-ac33-5354bfcdb6de, deleteOnExit=true 2024-12-08T05:45:25,898 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/cluster_8140fb43-a50a-3469-ac33-5354bfcdb6de/zookeeper_0, clientPort=56455, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/cluster_8140fb43-a50a-3469-ac33-5354bfcdb6de/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/cluster_8140fb43-a50a-3469-ac33-5354bfcdb6de/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T05:45:25,910 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56455 2024-12-08T05:45:25,911 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:26,005 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5 with version=8 2024-12-08T05:45:26,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/hbase-staging 2024-12-08T05:45:26,111 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T05:45:26,400 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0d942cb2025d:0 server-side Connection retries=45 2024-12-08T05:45:26,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T05:45:26,412 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T05:45:26,420 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T05:45:26,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T05:45:26,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T05:45:26,561 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T05:45:26,650 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T05:45:26,687 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T05:45:26,698 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T05:45:26,702 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T05:45:26,746 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14833 (auto-detected) 2024-12-08T05:45:26,747 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T05:45:26,775 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36947 2024-12-08T05:45:26,799 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36947 connecting to ZooKeeper ensemble=127.0.0.1:56455 2024-12-08T05:45:26,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369470x0, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T05:45:26,840 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36947-0x101909b07060000 connected 2024-12-08T05:45:26,868 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:26,868 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:26,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T05:45:26,882 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5, hbase.cluster.distributed=false 2024-12-08T05:45:26,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T05:45:26,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36947 2024-12-08T05:45:26,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36947 2024-12-08T05:45:26,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36947 2024-12-08T05:45:26,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36947 2024-12-08T05:45:26,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36947 2024-12-08T05:45:27,008 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d942cb2025d:0 server-side Connection retries=45 2024-12-08T05:45:27,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T05:45:27,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T05:45:27,010 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T05:45:27,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T05:45:27,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T05:45:27,014 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T05:45:27,017 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T05:45:27,018 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35457 2024-12-08T05:45:27,019 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35457 connecting to ZooKeeper ensemble=127.0.0.1:56455 2024-12-08T05:45:27,020 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:27,021 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:27,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354570x0, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T05:45:27,027 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35457-0x101909b07060001 connected 2024-12-08T05:45:27,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T05:45:27,032 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T05:45:27,048 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/bucket.cache 2024-12-08T05:45:27,053 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T05:45:27,055 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T05:45:27,056 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-12-08T05:45:27,058 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T05:45:27,058 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T05:45:27,063 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T05:45:27,066 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T05:45:27,067 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/bucket.persistence is complete. 2024-12-08T05:45:27,069 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T05:45:27,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T05:45:27,079 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T05:45:27,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35457 2024-12-08T05:45:27,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35457 2024-12-08T05:45:27,082 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35457 2024-12-08T05:45:27,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35457 2024-12-08T05:45:27,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35457 2024-12-08T05:45:27,110 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d942cb2025d:36947 2024-12-08T05:45:27,111 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d942cb2025d,36947,1733636726192 2024-12-08T05:45:27,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T05:45:27,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T05:45:27,122 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d942cb2025d,36947,1733636726192 2024-12-08T05:45:27,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T05:45:27,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,156 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T05:45:27,157 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d942cb2025d,36947,1733636726192 from backup master directory 2024-12-08T05:45:27,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d942cb2025d,36947,1733636726192 2024-12-08T05:45:27,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T05:45:27,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T05:45:27,162 WARN [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T05:45:27,162 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d942cb2025d,36947,1733636726192 2024-12-08T05:45:27,164 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T05:45:27,166 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T05:45:27,174 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/hbase.id] with ID: 3e788709-5134-4394-9359-19a91fa482a5 2024-12-08T05:45:27,174 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/.tmp/hbase.id 2024-12-08T05:45:27,180 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/hbase.id] 2024-12-08T05:45:27,227 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:27,232 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T05:45:27,260 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-12-08T05:45:27,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,293 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T05:45:27,297 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T05:45:27,344 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T05:45:27,351 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T05:45:27,415 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store 2024-12-08T05:45:27,436 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T05:45:27,440 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T05:45:27,441 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T05:45:27,441 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:27,442 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:27,443 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T05:45:27,443 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:27,444 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:27,444 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733636727441Disabling compacts and flushes for region at 1733636727441Disabling writes for close at 1733636727443 (+2 ms)Writing region close event to WAL at 1733636727444 (+1 ms)Closed at 1733636727444 2024-12-08T05:45:27,445 WARN [master/0d942cb2025d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/.initializing 2024-12-08T05:45:27,445 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/WALs/0d942cb2025d,36947,1733636726192 2024-12-08T05:45:27,458 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T05:45:27,471 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d942cb2025d%2C36947%2C1733636726192, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/WALs/0d942cb2025d,36947,1733636726192, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/oldWALs, maxLogs=10 2024-12-08T05:45:27,508 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/WALs/0d942cb2025d,36947,1733636726192/0d942cb2025d%2C36947%2C1733636726192.1733636727475 2024-12-08T05:45:27,509 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T05:45:27,510 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T05:45:27,510 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T05:45:27,514 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,515 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T05:45:27,592 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:27,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:27,594 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T05:45:27,600 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:27,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T05:45:27,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,606 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T05:45:27,607 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:27,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T05:45:27,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T05:45:27,613 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:27,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T05:45:27,614 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,617 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,618 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,621 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,621 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,624 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T05:45:27,626 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T05:45:27,634 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T05:45:27,635 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66840964, jitterRate=-0.003992021083831787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T05:45:27,641 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733636727528Initializing all the Stores at 1733636727528Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733636727529 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733636727529Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733636727530 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733636727530Cleaning up temporary data from old regions at 1733636727621 (+91 ms)Region opened successfully at 1733636727641 (+20 ms) 2024-12-08T05:45:27,642 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T05:45:27,675 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ba66f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d942cb2025d/172.17.0.2:0 2024-12-08T05:45:27,710 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T05:45:27,722 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T05:45:27,722 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T05:45:27,725 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T05:45:27,726 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-08T05:45:27,733 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-08T05:45:27,734 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T05:45:27,764 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T05:45:27,776 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T05:45:27,779 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T05:45:27,782 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T05:45:27,784 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T05:45:27,786 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T05:45:27,789 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T05:45:27,794 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T05:45:27,796 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T05:45:27,798 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T05:45:27,801 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T05:45:27,823 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T05:45:27,825 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T05:45:27,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T05:45:27,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T05:45:27,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,842 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d942cb2025d,36947,1733636726192, sessionid=0x101909b07060000, setting cluster-up flag (Was=false) 2024-12-08T05:45:27,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,862 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T05:45:27,864 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d942cb2025d,36947,1733636726192 2024-12-08T05:45:27,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:27,877 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T05:45:27,879 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d942cb2025d,36947,1733636726192 2024-12-08T05:45:27,884 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T05:45:27,889 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(746): ClusterId : 3e788709-5134-4394-9359-19a91fa482a5 2024-12-08T05:45:27,892 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T05:45:27,896 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T05:45:27,897 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T05:45:27,900 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T05:45:27,901 DEBUG [RS:0;0d942cb2025d:35457 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1222fc3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d942cb2025d/172.17.0.2:0 2024-12-08T05:45:27,916 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d942cb2025d:35457 2024-12-08T05:45:27,919 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T05:45:27,920 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T05:45:27,920 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T05:45:27,922 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d942cb2025d,36947,1733636726192 with port=35457, startcode=1733636726968 2024-12-08T05:45:27,934 DEBUG [RS:0;0d942cb2025d:35457 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T05:45:27,972 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T05:45:27,991 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T05:45:28,001 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T05:45:28,007 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50187, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T05:45:28,009 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d942cb2025d,36947,1733636726192 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T05:45:28,013 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36947 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T05:45:28,018 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d942cb2025d:0, corePoolSize=5, maxPoolSize=5 2024-12-08T05:45:28,019 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d942cb2025d:0, corePoolSize=5, maxPoolSize=5 2024-12-08T05:45:28,019 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d942cb2025d:0, corePoolSize=5, maxPoolSize=5 2024-12-08T05:45:28,019 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d942cb2025d:0, corePoolSize=5, maxPoolSize=5 2024-12-08T05:45:28,020 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d942cb2025d:0, corePoolSize=10, maxPoolSize=10 2024-12-08T05:45:28,020 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,020 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d942cb2025d:0, corePoolSize=2, maxPoolSize=2 2024-12-08T05:45:28,020 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,025 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T05:45:28,026 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T05:45:28,030 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,031 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T05:45:28,032 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733636758032 2024-12-08T05:45:28,034 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T05:45:28,035 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T05:45:28,040 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T05:45:28,040 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T05:45:28,041 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T05:45:28,041 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T05:45:28,045 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-08T05:45:28,045 WARN [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-08T05:45:28,044 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,048 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T05:45:28,049 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T05:45:28,050 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T05:45:28,056 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T05:45:28,057 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T05:45:28,058 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d942cb2025d:0:becomeActiveMaster-HFileCleaner.large.0-1733636728058,5,FailOnTimeoutGroup] 2024-12-08T05:45:28,059 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d942cb2025d:0:becomeActiveMaster-HFileCleaner.small.0-1733636728059,5,FailOnTimeoutGroup] 2024-12-08T05:45:28,059 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,059 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T05:45:28,060 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,061 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,062 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T05:45:28,062 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5 2024-12-08T05:45:28,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T05:45:28,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T05:45:28,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T05:45:28,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T05:45:28,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T05:45:28,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T05:45:28,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T05:45:28,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T05:45:28,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T05:45:28,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T05:45:28,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740 2024-12-08T05:45:28,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740 2024-12-08T05:45:28,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T05:45:28,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T05:45:28,095 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T05:45:28,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T05:45:28,104 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T05:45:28,104 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66594373, jitterRate=-0.0076665133237838745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T05:45:28,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733636728071Initializing all the Stores at 1733636728072 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733636728072Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733636728072Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733636728072Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733636728072Cleaning up temporary data from old regions at 1733636728095 (+23 ms)Region opened successfully at 1733636728109 (+14 ms) 2024-12-08T05:45:28,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T05:45:28,109 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T05:45:28,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T05:45:28,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T05:45:28,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T05:45:28,111 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T05:45:28,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733636728109Disabling compacts and flushes for region at 1733636728109Disabling writes for close at 1733636728110 (+1 ms)Writing region close event to WAL at 1733636728111 (+1 ms)Closed at 1733636728111 2024-12-08T05:45:28,115 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T05:45:28,115 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T05:45:28,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T05:45:28,130 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T05:45:28,132 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T05:45:28,147 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d942cb2025d,36947,1733636726192 with port=35457, startcode=1733636726968 2024-12-08T05:45:28,149 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36947 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,152 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36947 {}] master.ServerManager(517): Registering regionserver=0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,159 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5 2024-12-08T05:45:28,159 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T05:45:28,160 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T05:45:28,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T05:45:28,164 DEBUG [RS:0;0d942cb2025d:35457 {}] zookeeper.ZKUtil(111): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,165 WARN [RS:0;0d942cb2025d:35457 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T05:45:28,165 INFO [RS:0;0d942cb2025d:35457 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T05:45:28,165 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,167 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d942cb2025d,35457,1733636726968] 2024-12-08T05:45:28,191 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T05:45:28,205 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T05:45:28,210 INFO [RS:0;0d942cb2025d:35457 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T05:45:28,210 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,215 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T05:45:28,219 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T05:45:28,221 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,221 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,221 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,222 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,222 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,222 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,222 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d942cb2025d:0, corePoolSize=2, maxPoolSize=2 2024-12-08T05:45:28,222 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,223 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,223 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,223 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,223 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,224 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d942cb2025d:0, corePoolSize=1, maxPoolSize=1 2024-12-08T05:45:28,224 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d942cb2025d:0, corePoolSize=3, maxPoolSize=3 2024-12-08T05:45:28,224 DEBUG [RS:0;0d942cb2025d:35457 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d942cb2025d:0, corePoolSize=3, maxPoolSize=3 2024-12-08T05:45:28,225 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,226 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,226 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,226 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,226 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,226 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,35457,1733636726968-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T05:45:28,257 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T05:45:28,259 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,35457,1733636726968-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,260 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,260 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.Replication(171): 0d942cb2025d,35457,1733636726968 started 2024-12-08T05:45:28,281 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,281 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1482): Serving as 0d942cb2025d,35457,1733636726968, RpcServer on 0d942cb2025d/172.17.0.2:35457, sessionid=0x101909b07060001 2024-12-08T05:45:28,282 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T05:45:28,282 DEBUG [RS:0;0d942cb2025d:35457 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,283 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d942cb2025d,35457,1733636726968' 2024-12-08T05:45:28,283 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T05:45:28,283 WARN [0d942cb2025d:36947 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T05:45:28,284 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T05:45:28,285 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T05:45:28,285 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T05:45:28,285 DEBUG [RS:0;0d942cb2025d:35457 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,285 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d942cb2025d,35457,1733636726968' 2024-12-08T05:45:28,285 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T05:45:28,286 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T05:45:28,287 DEBUG [RS:0;0d942cb2025d:35457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T05:45:28,287 INFO [RS:0;0d942cb2025d:35457 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T05:45:28,287 INFO [RS:0;0d942cb2025d:35457 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T05:45:28,392 INFO [RS:0;0d942cb2025d:35457 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T05:45:28,393 INFO [RS:0;0d942cb2025d:35457 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0d942cb2025d%2C35457%2C1733636726968, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/oldWALs, maxLogs=55 2024-12-08T05:45:28,415 INFO [RS:0;0d942cb2025d:35457 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968/0d942cb2025d%2C35457%2C1733636726968.1733636728396 2024-12-08T05:45:28,415 DEBUG [RS:0;0d942cb2025d:35457 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T05:45:28,536 DEBUG [0d942cb2025d:36947 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T05:45:28,549 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,554 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d942cb2025d,35457,1733636726968, state=OPENING 2024-12-08T05:45:28,562 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T05:45:28,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:28,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:28,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T05:45:28,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T05:45:28,566 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T05:45:28,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d942cb2025d,35457,1733636726968}] 2024-12-08T05:45:28,741 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T05:45:28,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41207, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T05:45:28,752 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T05:45:28,752 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T05:45:28,753 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T05:45:28,754 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0d942cb2025d%2C35457%2C1733636726968.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/oldWALs, maxLogs=55 2024-12-08T05:45:28,775 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968/0d942cb2025d%2C35457%2C1733636726968.meta.1733636728756.meta 2024-12-08T05:45:28,775 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T05:45:28,775 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T05:45:28,777 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T05:45:28,780 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T05:45:28,784 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T05:45:28,788 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T05:45:28,789 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T05:45:28,789 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T05:45:28,789 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T05:45:28,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T05:45:28,792 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T05:45:28,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T05:45:28,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T05:45:28,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T05:45:28,795 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T05:45:28,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T05:45:28,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T05:45:28,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:28,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T05:45:28,797 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T05:45:28,798 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740 2024-12-08T05:45:28,799 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740 2024-12-08T05:45:28,799 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T05:45:28,800 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T05:45:28,800 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T05:45:28,801 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T05:45:28,802 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62250267, jitterRate=-0.07239873707294464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T05:45:28,802 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T05:45:28,803 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733636728790Writing region info on filesystem at 1733636728790Initializing all the Stores at 1733636728790Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733636728790Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733636728791 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733636728791Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733636728791Cleaning up temporary data from old regions at 1733636728800 (+9 ms)Running coprocessor post-open hooks at 1733636728802 (+2 ms)Region opened successfully at 1733636728803 (+1 ms) 2024-12-08T05:45:28,809 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733636728732 2024-12-08T05:45:28,819 DEBUG [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T05:45:28,819 INFO [RS_OPEN_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T05:45:28,820 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,821 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d942cb2025d,35457,1733636726968, state=OPEN 2024-12-08T05:45:28,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T05:45:28,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T05:45:28,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T05:45:28,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T05:45:28,829 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d942cb2025d,35457,1733636726968 2024-12-08T05:45:28,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T05:45:28,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d942cb2025d,35457,1733636726968 in 261 msec 2024-12-08T05:45:28,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T05:45:28,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 711 msec 2024-12-08T05:45:28,837 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T05:45:28,837 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T05:45:28,858 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T05:45:28,860 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d942cb2025d,35457,1733636726968, seqNum=-1] 2024-12-08T05:45:28,879 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T05:45:28,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T05:45:28,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 978 msec 2024-12-08T05:45:28,897 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733636728897, completionTime=-1 2024-12-08T05:45:28,900 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T05:45:28,900 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T05:45:28,926 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T05:45:28,926 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733636788926 2024-12-08T05:45:28,927 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733636848926 2024-12-08T05:45:28,927 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-08T05:45:28,929 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,36947,1733636726192-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,930 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,36947,1733636726192-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,930 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,36947,1733636726192-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,931 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d942cb2025d:36947, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,932 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,932 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,937 DEBUG [master/0d942cb2025d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T05:45:28,957 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.795sec 2024-12-08T05:45:28,958 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T05:45:28,959 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T05:45:28,961 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T05:45:28,961 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T05:45:28,961 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T05:45:28,962 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,36947,1733636726192-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T05:45:28,963 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,36947,1733636726192-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T05:45:28,967 DEBUG [master/0d942cb2025d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T05:45:28,968 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T05:45:28,968 INFO [master/0d942cb2025d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d942cb2025d,36947,1733636726192-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T05:45:28,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1b595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T05:45:29,001 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T05:45:29,001 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T05:45:29,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d942cb2025d,36947,-1 for getting cluster id 2024-12-08T05:45:29,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T05:45:29,016 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3e788709-5134-4394-9359-19a91fa482a5' 2024-12-08T05:45:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T05:45:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3e788709-5134-4394-9359-19a91fa482a5" 2024-12-08T05:45:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e63efe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T05:45:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d942cb2025d,36947,-1] 2024-12-08T05:45:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T05:45:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T05:45:29,031 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T05:45:29,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5a12a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T05:45:29,034 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T05:45:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d942cb2025d,35457,1733636726968, seqNum=-1] 2024-12-08T05:45:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T05:45:29,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T05:45:29,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d942cb2025d,36947,1733636726192 2024-12-08T05:45:29,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T05:45:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0d942cb2025d,36947,1733636726192 2024-12-08T05:45:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44da81de 2024-12-08T05:45:29,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T05:45:29,090 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T05:45:29,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T05:45:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-12-08T05:45:29,107 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T05:45:29,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-12-08T05:45:29,110 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:29,111 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T05:45:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T05:45:29,126 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a051167ebbbfb6f831dbb96a8adf735, NAME => 'table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5 2024-12-08T05:45:29,133 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T05:45:29,133 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 1a051167ebbbfb6f831dbb96a8adf735, disabling compactions & flushes 2024-12-08T05:45:29,133 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:29,133 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:29,133 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. after waiting 0 ms 2024-12-08T05:45:29,133 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:29,133 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:29,133 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a051167ebbbfb6f831dbb96a8adf735: Waiting for close lock at 1733636729133Disabling compacts and flushes for region at 1733636729133Disabling writes for close at 1733636729133Writing region close event to WAL at 1733636729133Closed at 1733636729133 2024-12-08T05:45:29,135 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T05:45:29,140 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1733636729135"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733636729135"}]},"ts":"1733636729135"} 2024-12-08T05:45:29,144 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T05:45:29,145 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T05:45:29,148 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733636729146"}]},"ts":"1733636729146"} 2024-12-08T05:45:29,152 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-12-08T05:45:29,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=1a051167ebbbfb6f831dbb96a8adf735, ASSIGN}] 2024-12-08T05:45:29,156 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=1a051167ebbbfb6f831dbb96a8adf735, ASSIGN 2024-12-08T05:45:29,157 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=1a051167ebbbfb6f831dbb96a8adf735, ASSIGN; state=OFFLINE, location=0d942cb2025d,35457,1733636726968; forceNewPlan=false, retain=false 2024-12-08T05:45:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T05:45:29,308 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a051167ebbbfb6f831dbb96a8adf735, regionState=OPENING, regionLocation=0d942cb2025d,35457,1733636726968 2024-12-08T05:45:29,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=1a051167ebbbfb6f831dbb96a8adf735, ASSIGN because future has completed 2024-12-08T05:45:29,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a051167ebbbfb6f831dbb96a8adf735, server=0d942cb2025d,35457,1733636726968}] 2024-12-08T05:45:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T05:45:29,468 INFO [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:29,469 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a051167ebbbfb6f831dbb96a8adf735, NAME => 'table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735.', STARTKEY => '', ENDKEY => ''} 2024-12-08T05:45:29,469 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,469 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T05:45:29,470 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,470 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,471 INFO [StoreOpener-1a051167ebbbfb6f831dbb96a8adf735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,479 INFO [StoreOpener-1a051167ebbbfb6f831dbb96a8adf735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a051167ebbbfb6f831dbb96a8adf735 columnFamilyName family 2024-12-08T05:45:29,479 DEBUG [StoreOpener-1a051167ebbbfb6f831dbb96a8adf735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T05:45:29,480 INFO [StoreOpener-1a051167ebbbfb6f831dbb96a8adf735-1 {}] regionserver.HStore(327): Store=1a051167ebbbfb6f831dbb96a8adf735/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T05:45:29,480 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,481 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,481 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,481 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,481 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,482 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,488 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T05:45:29,489 INFO [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1a051167ebbbfb6f831dbb96a8adf735; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58727717, jitterRate=-0.1248888224363327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T05:45:29,489 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:29,490 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1a051167ebbbfb6f831dbb96a8adf735: Running coprocessor pre-open hook at 1733636729470Writing region info on filesystem at 1733636729470Initializing all the Stores at 1733636729470Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733636729470Cleaning up temporary data from old regions at 1733636729481 (+11 ms)Running coprocessor post-open hooks at 1733636729489 (+8 ms)Region opened successfully at 1733636729490 (+1 ms) 2024-12-08T05:45:29,491 INFO [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735., pid=6, masterSystemTime=1733636729465 2024-12-08T05:45:29,494 DEBUG [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:29,494 INFO [RS_OPEN_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:29,494 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a051167ebbbfb6f831dbb96a8adf735, regionState=OPEN, openSeqNum=2, regionLocation=0d942cb2025d,35457,1733636726968 2024-12-08T05:45:29,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a051167ebbbfb6f831dbb96a8adf735, server=0d942cb2025d,35457,1733636726968 because future has completed 2024-12-08T05:45:29,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T05:45:29,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a051167ebbbfb6f831dbb96a8adf735, server=0d942cb2025d,35457,1733636726968 in 186 msec 2024-12-08T05:45:29,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T05:45:29,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=1a051167ebbbfb6f831dbb96a8adf735, ASSIGN in 346 msec 2024-12-08T05:45:29,504 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T05:45:29,504 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733636729504"}]},"ts":"1733636729504"} 2024-12-08T05:45:29,507 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-12-08T05:45:29,508 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T05:45:29,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 407 msec 2024-12-08T05:45:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T05:45:29,749 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-12-08T05:45:29,749 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-12-08T05:45:29,752 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T05:45:29,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-12-08T05:45:29,758 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T05:45:29,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-12-08T05:45:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735., hostname=0d942cb2025d,35457,1733636726968, seqNum=2] 2024-12-08T05:45:29,777 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a051167ebbbfb6f831dbb96a8adf735 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-08T05:45:29,802 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 008df6af58e04cb887390fec4d6d38eb 2024-12-08T05:45:29,838 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/.tmp/family/008df6af58e04cb887390fec4d6d38eb is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-08T05:45:29,843 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/.tmp/family/008df6af58e04cb887390fec4d6d38eb 2024-12-08T05:45:29,878 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/.tmp/family/008df6af58e04cb887390fec4d6d38eb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb 2024-12-08T05:45:29,879 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb, delay=977 ms 2024-12-08T05:45:29,881 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb, entries=2, sequenceid=6, filesize=4.9 K 2024-12-08T05:45:29,888 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1a051167ebbbfb6f831dbb96a8adf735 in 109ms, sequenceid=6, compaction requested=false 2024-12-08T05:45:29,889 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-12-08T05:45:29,890 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1a051167ebbbfb6f831dbb96a8adf735: 2024-12-08T05:45:30,859 DEBUG [hfile-prefetch-1733636729879 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 008df6af58e04cb887390fec4d6d38eb_0. 2024-12-08T05:45:30,863 DEBUG [hfile-prefetch-1733636729879 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 008df6af58e04cb887390fec4d6d38eb_117. 2024-12-08T05:45:30,864 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb, with total blocks 2, and data blocks 1 2024-12-08T05:45:30,864 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-08T05:45:30,864 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-08T05:45:30,865 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb. Will sleep 100ms and try the verification again. 2024-12-08T05:45:30,965 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb, with total blocks 2, and data blocks 1 2024-12-08T05:45:30,965 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-08T05:45:30,965 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2293): found block for file 008df6af58e04cb887390fec4d6d38eb in the backing map. Acquiring read lock for offset 0 2024-12-08T05:45:30,965 DEBUG [hfile-prefetch-1733636729879 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/family/008df6af58e04cb887390fec4d6d38eb has now been fully cached. 2024-12-08T05:45:30,966 DEBUG [hfile-prefetch-1733636729879 {}] hfile.PrefetchExecutor(121): Prefetch completed for 008df6af58e04cb887390fec4d6d38eb 2024-12-08T05:45:31,057 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T05:45:31,057 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T05:45:31,058 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T05:45:31,058 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-08T05:45:31,078 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-08T05:45:31,079 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733636731057 2024-12-08T05:45:31,079 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T05:45:31,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T05:45:31,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T05:45:31,899 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;0d942cb2025d:35457,5,FailOnTimeoutGroup] 2024-12-08T05:45:31,899 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d942cb2025d,35457,1733636726968' ***** 2024-12-08T05:45:31,900 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-08T05:45:31,900 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-12-08T05:45:31,900 INFO [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T05:45:31,900 INFO [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/bucket.persistence 2024-12-08T05:45:31,900 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T05:45:31,900 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T05:45:31,900 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T05:45:31,901 INFO [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T05:45:31,901 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T05:45:31,911 DEBUG [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1487): Thread RS:0;0d942cb2025d:35457 started persisting bucket cache to file 2024-12-08T05:45:31,911 DEBUG [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T05:45:31,911 DEBUG [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-08T05:45:31,914 DEBUG [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-08T05:45:31,915 DEBUG [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733636731911 2024-12-08T05:45:31,915 DEBUG [RS:0;0d942cb2025d:35457 {}] bucket.BucketCache(1509): Thread RS:0;0d942cb2025d:35457 finished persisting bucket cache to file, renaming 2024-12-08T05:45:31,915 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T05:45:31,915 INFO [RS:0;0d942cb2025d:35457 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T05:45:31,915 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T05:45:31,915 INFO [RS:0;0d942cb2025d:35457 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T05:45:31,916 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:31,916 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(959): stopping server 0d942cb2025d,35457,1733636726968 2024-12-08T05:45:31,916 INFO [RS:0;0d942cb2025d:35457 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T05:45:31,916 INFO [RS:0;0d942cb2025d:35457 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d942cb2025d:35457. 2024-12-08T05:45:31,916 INFO [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T05:45:31,917 DEBUG [RS:0;0d942cb2025d:35457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T05:45:31,917 DEBUG [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a051167ebbbfb6f831dbb96a8adf735, disabling compactions & flushes 2024-12-08T05:45:31,917 INFO [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:31,917 DEBUG [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:31,917 DEBUG [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. after waiting 0 ms 2024-12-08T05:45:31,917 DEBUG [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:31,922 DEBUG [RS:0;0d942cb2025d:35457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T05:45:31,922 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T05:45:31,922 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T05:45:31,922 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T05:45:31,923 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T05:45:31,923 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-08T05:45:31,923 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T05:45:31,924 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1a051167ebbbfb6f831dbb96a8adf735=table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735.} 2024-12-08T05:45:31,924 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T05:45:31,924 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T05:45:31,924 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T05:45:31,924 DEBUG [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a051167ebbbfb6f831dbb96a8adf735 2024-12-08T05:45:31,924 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T05:45:31,924 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T05:45:31,924 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-12-08T05:45:31,924 DEBUG [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/default/table1/1a051167ebbbfb6f831dbb96a8adf735/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-08T05:45:31,926 INFO [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:31,926 DEBUG [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a051167ebbbfb6f831dbb96a8adf735: Waiting for close lock at 1733636731916Running coprocessor pre-close hooks at 1733636731917 (+1 ms)Disabling compacts and flushes for region at 1733636731917Disabling writes for close at 1733636731917Writing region close event to WAL at 1733636731919 (+2 ms)Running coprocessor post-close hooks at 1733636731925 (+6 ms)Closed at 1733636731926 (+1 ms) 2024-12-08T05:45:31,926 DEBUG [RS_CLOSE_REGION-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735. 2024-12-08T05:45:31,930 INFO [regionserver/0d942cb2025d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T05:45:31,935 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6789ce010bc14b03b800bcda2138b166 2024-12-08T05:45:31,952 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/info/6789ce010bc14b03b800bcda2138b166 is 133, key is table1,,1733636729092.1a051167ebbbfb6f831dbb96a8adf735./info:regioninfo/1733636729494/Put/seqid=0 2024-12-08T05:45:31,953 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/info/6789ce010bc14b03b800bcda2138b166 2024-12-08T05:45:31,962 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 83f1d89028ad411382062b3771344a4b 2024-12-08T05:45:31,977 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/ns/83f1d89028ad411382062b3771344a4b is 43, key is default/ns:d/1733636728886/Put/seqid=0 2024-12-08T05:45:31,977 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/ns/83f1d89028ad411382062b3771344a4b 2024-12-08T05:45:31,986 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fc21c7d65c6e4c61b7325b60d6c716dc 2024-12-08T05:45:32,000 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/table/fc21c7d65c6e4c61b7325b60d6c716dc is 42, key is table1/table:state/1733636729504/Put/seqid=0 2024-12-08T05:45:32,000 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/table/fc21c7d65c6e4c61b7325b60d6c716dc 2024-12-08T05:45:32,003 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/info/6789ce010bc14b03b800bcda2138b166 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/info/6789ce010bc14b03b800bcda2138b166 2024-12-08T05:45:32,004 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/info/6789ce010bc14b03b800bcda2138b166, delay=919 ms 2024-12-08T05:45:32,005 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/info/6789ce010bc14b03b800bcda2138b166, entries=10, sequenceid=11, filesize=6.3 K 2024-12-08T05:45:32,006 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/ns/83f1d89028ad411382062b3771344a4b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/ns/83f1d89028ad411382062b3771344a4b 2024-12-08T05:45:32,007 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/ns/83f1d89028ad411382062b3771344a4b, delay=997 ms 2024-12-08T05:45:32,008 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/ns/83f1d89028ad411382062b3771344a4b, entries=2, sequenceid=11, filesize=5.0 K 2024-12-08T05:45:32,009 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/.tmp/table/fc21c7d65c6e4c61b7325b60d6c716dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/table/fc21c7d65c6e4c61b7325b60d6c716dc 2024-12-08T05:45:32,010 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/table/fc21c7d65c6e4c61b7325b60d6c716dc, delay=981 ms 2024-12-08T05:45:32,011 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/table/fc21c7d65c6e4c61b7325b60d6c716dc, entries=2, sequenceid=11, filesize=5.1 K 2024-12-08T05:45:32,012 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=11, compaction requested=false 2024-12-08T05:45:32,012 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T05:45:32,014 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/info/6789ce010bc14b03b800bcda2138b166 2024-12-08T05:45:32,015 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/ns/83f1d89028ad411382062b3771344a4b 2024-12-08T05:45:32,015 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/table/fc21c7d65c6e4c61b7325b60d6c716dc 2024-12-08T05:45:32,020 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-08T05:45:32,020 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T05:45:32,021 INFO [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T05:45:32,021 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733636731923Running coprocessor pre-close hooks at 1733636731924 (+1 ms)Disabling compacts and flushes for region at 1733636731924Disabling writes for close at 1733636731924Obtaining lock to block concurrent updates at 1733636731924Preparing flush snapshotting stores in 1588230740 at 1733636731924Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1733636731926 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733636731926Flushing 1588230740/info: creating writer at 1733636731926Flushing 1588230740/info: appending metadata at 1733636731951 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733636731951Flushing 1588230740/ns: creating writer at 1733636731956 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733636731975 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733636731975Flushing 1588230740/table: creating writer at 1733636731980 (+5 ms)Flushing 1588230740/table: appending metadata at 1733636731999 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733636731999Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a79ee9: reopening flushed file at 1733636732003 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a704ffc: reopening flushed file at 1733636732006 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15865bb0: reopening flushed file at 1733636732009 (+3 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=11, compaction requested=false at 1733636732012 (+3 ms)Writing region close event to WAL at 1733636732016 (+4 ms)Running coprocessor post-close hooks at 1733636732020 (+4 ms)Closed at 1733636732020 2024-12-08T05:45:32,021 DEBUG [RS_CLOSE_META-regionserver/0d942cb2025d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T05:45:32,124 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(976): stopping server 0d942cb2025d,35457,1733636726968; all regions closed. 2024-12-08T05:45:32,129 DEBUG [RS:0;0d942cb2025d:35457 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/oldWALs 2024-12-08T05:45:32,129 INFO [RS:0;0d942cb2025d:35457 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0d942cb2025d%2C35457%2C1733636726968.meta:.meta(num 1733636728756) 2024-12-08T05:45:32,131 DEBUG [RS:0;0d942cb2025d:35457 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/oldWALs 2024-12-08T05:45:32,131 INFO [RS:0;0d942cb2025d:35457 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0d942cb2025d%2C35457%2C1733636726968:(num 1733636728396) 2024-12-08T05:45:32,131 DEBUG [RS:0;0d942cb2025d:35457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T05:45:32,131 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T05:45:32,131 INFO [RS:0;0d942cb2025d:35457 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T05:45:32,132 INFO [RS:0;0d942cb2025d:35457 {}] hbase.ChoreService(370): Chore service for: regionserver/0d942cb2025d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T05:45:32,132 INFO [RS:0;0d942cb2025d:35457 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T05:45:32,132 INFO [regionserver/0d942cb2025d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T05:45:32,132 INFO [RS:0;0d942cb2025d:35457 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35457 2024-12-08T05:45:32,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d942cb2025d,35457,1733636726968 2024-12-08T05:45:32,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T05:45:32,138 INFO [RS:0;0d942cb2025d:35457 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T05:45:32,139 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d942cb2025d,35457,1733636726968] 2024-12-08T05:45:32,141 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d942cb2025d,35457,1733636726968 already deleted, retry=false 2024-12-08T05:45:32,141 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0d942cb2025d,35457,1733636726968 on 0d942cb2025d,36947,1733636726192 2024-12-08T05:45:32,145 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0d942cb2025d,35457,1733636726968, splitWal=true, meta=true 2024-12-08T05:45:32,148 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0d942cb2025d,35457,1733636726968, splitWal=true, meta=true 2024-12-08T05:45:32,148 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 0d942cb2025d,35457,1733636726968 (carryingMeta=true) 0d942cb2025d,35457,1733636726968/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@684ca9b0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-08T05:45:32,149 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 0d942cb2025d,35457,1733636726968, splitWal=true, meta=true, isMeta: true 2024-12-08T05:45:32,150 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968-splitting 2024-12-08T05:45:32,150 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968-splitting dir is empty, no logs to split. 2024-12-08T05:45:32,150 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0d942cb2025d,35457,1733636726968 WAL count=0, meta=true 2024-12-08T05:45:32,152 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/WALs/0d942cb2025d,35457,1733636726968-splitting dir is empty, no logs to split. 2024-12-08T05:45:32,152 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0d942cb2025d,35457,1733636726968 WAL count=0, meta=true 2024-12-08T05:45:32,152 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0d942cb2025d,35457,1733636726968 WAL splitting is done? wals=0, meta=true 2024-12-08T05:45:32,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T05:45:32,154 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T05:45:32,155 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T05:45:32,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T05:45:32,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35457-0x101909b07060001, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T05:45:32,241 INFO [RS:0;0d942cb2025d:35457 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T05:45:32,241 INFO [RS:0;0d942cb2025d:35457 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d942cb2025d,35457,1733636726968; zookeeper connection closed. 2024-12-08T05:45:32,242 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5629dc10 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5629dc10 2024-12-08T05:45:32,305 WARN [0d942cb2025d:36947 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T05:45:32,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T05:45:32,900 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T05:45:32,900 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T05:45:32,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T05:45:32,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T05:45:32,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T05:45:32,901 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T05:45:32,901 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=365092990, stopped=false 2024-12-08T05:45:32,902 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d942cb2025d,36947,1733636726192 2024-12-08T05:45:32,902 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0d942cb2025d,36947,1733636726192' ***** 2024-12-08T05:45:32,902 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-12-08T05:45:32,902 INFO [M:0;0d942cb2025d:36947 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T05:45:32,902 INFO [M:0;0d942cb2025d:36947 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;0d942cb2025d:36947. 2024-12-08T05:45:32,902 DEBUG [M:0;0d942cb2025d:36947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T05:45:32,902 DEBUG [M:0;0d942cb2025d:36947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T05:45:32,902 INFO [M:0;0d942cb2025d:36947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T05:45:32,903 DEBUG [M:0;0d942cb2025d:36947 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T05:45:32,903 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T05:45:32,903 DEBUG [M:0;0d942cb2025d:36947 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T05:45:32,903 DEBUG [master/0d942cb2025d:0:becomeActiveMaster-HFileCleaner.small.0-1733636728059 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d942cb2025d:0:becomeActiveMaster-HFileCleaner.small.0-1733636728059,5,FailOnTimeoutGroup] 2024-12-08T05:45:32,903 DEBUG [master/0d942cb2025d:0:becomeActiveMaster-HFileCleaner.large.0-1733636728058 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d942cb2025d:0:becomeActiveMaster-HFileCleaner.large.0-1733636728058,5,FailOnTimeoutGroup] 2024-12-08T05:45:32,903 INFO [M:0;0d942cb2025d:36947 {}] hbase.ChoreService(370): Chore service for: master/0d942cb2025d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T05:45:32,903 INFO [M:0;0d942cb2025d:36947 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T05:45:32,903 DEBUG [M:0;0d942cb2025d:36947 {}] master.HMaster(1795): Stopping service threads 2024-12-08T05:45:32,903 INFO [M:0;0d942cb2025d:36947 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T05:45:32,904 INFO [M:0;0d942cb2025d:36947 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T05:45:32,904 INFO [M:0;0d942cb2025d:36947 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T05:45:32,905 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T05:45:32,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T05:45:32,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T05:45:32,906 DEBUG [M:0;0d942cb2025d:36947 {}] zookeeper.ZKUtil(347): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T05:45:32,906 WARN [M:0;0d942cb2025d:36947 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T05:45:32,906 INFO [M:0;0d942cb2025d:36947 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/.lastflushedseqids 2024-12-08T05:45:32,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T05:45:32,908 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T05:45:32,908 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T05:45:32,919 INFO [M:0;0d942cb2025d:36947 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T05:45:33,056 DEBUG [0d942cb2025d:36947 {}] assignment.AssignmentManager(2455): Stopped! Dropping assign of 1 queued regions. 2024-12-08T05:45:33,056 INFO [M:0;0d942cb2025d:36947 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T05:45:33,057 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T05:45:33,057 INFO [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:33,057 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:33,057 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T05:45:33,057 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:33,057 INFO [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.18 KB heapSize=30.80 KB 2024-12-08T05:45:33,068 DEBUG [M:0;0d942cb2025d:36947 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on efd969d6c7ab4e068f7378630e500cf0 2024-12-08T05:45:33,082 DEBUG [M:0;0d942cb2025d:36947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efd969d6c7ab4e068f7378630e500cf0 is 82, key is hbase:meta,,1/info:regioninfo/1733636728820/Put/seqid=0 2024-12-08T05:45:33,082 INFO [M:0;0d942cb2025d:36947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efd969d6c7ab4e068f7378630e500cf0 2024-12-08T05:45:33,091 DEBUG [M:0;0d942cb2025d:36947 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1e40ef60a4884cc79a050a83f5e5707b 2024-12-08T05:45:33,104 DEBUG [M:0;0d942cb2025d:36947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e40ef60a4884cc79a050a83f5e5707b is 707, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733636729509/Put/seqid=0 2024-12-08T05:45:33,105 INFO [M:0;0d942cb2025d:36947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.57 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e40ef60a4884cc79a050a83f5e5707b 2024-12-08T05:45:33,113 DEBUG [M:0;0d942cb2025d:36947 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 88e3114e8c324aa3af820bb73f331fd7 2024-12-08T05:45:33,126 DEBUG [M:0;0d942cb2025d:36947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/88e3114e8c324aa3af820bb73f331fd7 is 58, key is 0d942cb2025d,35457,1733636726968/rs:/1733636732149/DeleteFamily/seqid=0 2024-12-08T05:45:33,127 INFO [M:0;0d942cb2025d:36947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/88e3114e8c324aa3af820bb73f331fd7 2024-12-08T05:45:33,129 INFO [M:0;0d942cb2025d:36947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88e3114e8c324aa3af820bb73f331fd7 2024-12-08T05:45:33,129 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efd969d6c7ab4e068f7378630e500cf0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efd969d6c7ab4e068f7378630e500cf0 2024-12-08T05:45:33,131 INFO [M:0;0d942cb2025d:36947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efd969d6c7ab4e068f7378630e500cf0, entries=8, sequenceid=64, filesize=5.5 K 2024-12-08T05:45:33,131 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e40ef60a4884cc79a050a83f5e5707b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e40ef60a4884cc79a050a83f5e5707b 2024-12-08T05:45:33,133 INFO [M:0;0d942cb2025d:36947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e40ef60a4884cc79a050a83f5e5707b, entries=8, sequenceid=64, filesize=6.5 K 2024-12-08T05:45:33,133 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/88e3114e8c324aa3af820bb73f331fd7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/88e3114e8c324aa3af820bb73f331fd7 2024-12-08T05:45:33,135 INFO [M:0;0d942cb2025d:36947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88e3114e8c324aa3af820bb73f331fd7 2024-12-08T05:45:33,135 INFO [M:0;0d942cb2025d:36947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77f20822-fc62-4624-4c82-e0b977334976/df2b335c-0ce1-3625-edc0-375bb92827b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/88e3114e8c324aa3af820bb73f331fd7, entries=1, sequenceid=64, filesize=5.2 K 2024-12-08T05:45:33,136 INFO [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=64, compaction requested=false 2024-12-08T05:45:33,138 INFO [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T05:45:33,138 DEBUG [M:0;0d942cb2025d:36947 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733636733057Disabling compacts and flushes for region at 1733636733057Disabling writes for close at 1733636733057Obtaining lock to block concurrent updates at 1733636733057Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733636733057Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24759, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1733636733058 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733636733058Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733636733058Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733636733081 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733636733081Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733636733084 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733636733103 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733636733103Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733636733106 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733636733125 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733636733125Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1de44d1d: reopening flushed file at 1733636733129 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19809bf9: reopening flushed file at 1733636733131 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4581a8ee: reopening flushed file at 1733636733133 (+2 ms)Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=64, compaction requested=false at 1733636733136 (+3 ms)Writing region close event to WAL at 1733636733138 (+2 ms)Closed at 1733636733138 2024-12-08T05:45:33,140 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T05:45:33,140 INFO [M:0;0d942cb2025d:36947 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T05:45:33,140 INFO [M:0;0d942cb2025d:36947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36947 2024-12-08T05:45:33,140 INFO [M:0;0d942cb2025d:36947 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T05:45:33,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T05:45:33,243 INFO [M:0;0d942cb2025d:36947 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T05:45:33,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36947-0x101909b07060000, quorum=127.0.0.1:56455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T05:45:33,247 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T05:45:33,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-08T05:45:33,269 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T05:45:33,277 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0d942cb2025d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733636732007 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0d942cb2025d:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733636729879 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733636732004 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0d942cb2025d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1733636732010 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 334), ProcessCount=11 (was 11), AvailableMemoryMB=8164 (was 8463)