2024-12-05 14:08:02,212 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-12-05 14:08:02,228 main DEBUG Took 0.014517 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 14:08:02,229 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 14:08:02,229 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 14:08:02,230 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 14:08:02,232 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,240 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 14:08:02,255 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,257 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,257 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,258 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,259 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,260 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,260 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,261 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,262 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,262 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,263 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,263 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:08:02,264 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,264 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 14:08:02,265 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:08:02,266 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 14:08:02,268 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 14:08:02,268 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 14:08:02,269 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 14:08:02,270 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 14:08:02,277 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 14:08:02,280 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 14:08:02,281 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 14:08:02,282 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 14:08:02,282 main DEBUG createAppenders(={Console}) 2024-12-05 14:08:02,283 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-12-05 14:08:02,283 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-12-05 14:08:02,284 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-12-05 14:08:02,284 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 14:08:02,284 main DEBUG OutputStream closed 2024-12-05 14:08:02,284 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 14:08:02,285 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 14:08:02,285 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-12-05 14:08:02,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 14:08:02,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 14:08:02,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 14:08:02,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 14:08:02,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 14:08:02,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 14:08:02,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 14:08:02,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 14:08:02,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 14:08:02,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 14:08:02,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 14:08:02,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 14:08:02,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 14:08:02,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 14:08:02,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 14:08:02,368 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 14:08:02,368 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 14:08:02,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 14:08:02,370 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 14:08:02,371 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-12-05 14:08:02,371 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 14:08:02,372 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-12-05T14:08:02,383 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-12-05 14:08:02,386 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 14:08:02,386 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T14:08:02,597 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4 2024-12-05T14:08:02,622 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=8806 2024-12-05T14:08:02,701 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T14:08:02,781 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:02,796 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/cluster_0504f6b0-70e6-368d-c129-bb6315e816f8, deleteOnExit=true 2024-12-05T14:08:02,885 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/cluster_0504f6b0-70e6-368d-c129-bb6315e816f8/zookeeper_0, clientPort=57001, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/cluster_0504f6b0-70e6-368d-c129-bb6315e816f8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/cluster_0504f6b0-70e6-368d-c129-bb6315e816f8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T14:08:02,897 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57001 2024-12-05T14:08:02,898 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:02,965 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c with version=8 2024-12-05T14:08:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/hbase-staging 2024-12-05T14:08:03,043 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T14:08:03,275 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da6aa2204f50:0 server-side Connection retries=45 2024-12-05T14:08:03,283 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:08:03,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T14:08:03,290 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T14:08:03,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:08:03,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T14:08:03,398 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T14:08:03,466 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T14:08:03,500 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T14:08:03,508 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T14:08:03,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T14:08:03,540 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 115566 (auto-detected) 2024-12-05T14:08:03,541 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T14:08:03,557 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46407 2024-12-05T14:08:03,576 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46407 connecting to ZooKeeper ensemble=127.0.0.1:57001 2024-12-05T14:08:03,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464070x0, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T14:08:03,682 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46407-0x101a71bd3a40000 connected 2024-12-05T14:08:03,756 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:03,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:03,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:08:03,773 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c, hbase.cluster.distributed=false 2024-12-05T14:08:03,796 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T14:08:03,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46407 2024-12-05T14:08:03,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46407 2024-12-05T14:08:03,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46407 2024-12-05T14:08:03,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46407 2024-12-05T14:08:03,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46407 2024-12-05T14:08:03,897 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da6aa2204f50:0 server-side Connection retries=45 2024-12-05T14:08:03,899 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:08:03,899 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T14:08:03,899 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T14:08:03,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:08:03,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T14:08:03,903 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T14:08:03,906 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T14:08:03,907 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34743 2024-12-05T14:08:03,908 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34743 connecting to ZooKeeper ensemble=127.0.0.1:57001 2024-12-05T14:08:03,909 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:03,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:03,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347430x0, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T14:08:03,916 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:347430x0, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:08:03,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34743-0x101a71bd3a40001 connected 2024-12-05T14:08:03,922 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T14:08:03,933 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/bucket.cache 2024-12-05T14:08:03,938 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T14:08:03,939 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T14:08:03,941 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-12-05T14:08:03,947 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T14:08:03,947 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T14:08:03,954 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T14:08:03,957 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T14:08:03,957 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/bucket.persistence is complete. 2024-12-05T14:08:03,960 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T14:08:03,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T14:08:03,969 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T14:08:03,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34743 2024-12-05T14:08:03,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34743 2024-12-05T14:08:03,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34743 2024-12-05T14:08:03,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34743 2024-12-05T14:08:03,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34743 2024-12-05T14:08:04,000 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da6aa2204f50:46407 2024-12-05T14:08:04,001 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da6aa2204f50,46407,1733407683113 2024-12-05T14:08:04,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:08:04,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:08:04,018 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da6aa2204f50,46407,1733407683113 2024-12-05T14:08:04,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T14:08:04,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,050 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T14:08:04,051 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da6aa2204f50,46407,1733407683113 from backup master directory 2024-12-05T14:08:04,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da6aa2204f50,46407,1733407683113 2024-12-05T14:08:04,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:08:04,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:08:04,058 WARN [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T14:08:04,059 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da6aa2204f50,46407,1733407683113 2024-12-05T14:08:04,061 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T14:08:04,063 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T14:08:04,072 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/hbase.id] with ID: 7d1a65e7-77b1-485d-b2ac-464e96ef5ab4 2024-12-05T14:08:04,072 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/.tmp/hbase.id 2024-12-05T14:08:04,078 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/hbase.id] 2024-12-05T14:08:04,110 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:04,114 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T14:08:04,132 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-05T14:08:04,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,165 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T14:08:04,168 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T14:08:04,205 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T14:08:04,210 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:08:04,256 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store 2024-12-05T14:08:04,273 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T14:08:04,276 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:08:04,278 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T14:08:04,278 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:04,278 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:04,280 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T14:08:04,280 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:04,280 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:04,281 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733407684277Disabling compacts and flushes for region at 1733407684277Disabling writes for close at 1733407684280 (+3 ms)Writing region close event to WAL at 1733407684280Closed at 1733407684280 2024-12-05T14:08:04,282 WARN [master/da6aa2204f50:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/.initializing 2024-12-05T14:08:04,282 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/WALs/da6aa2204f50,46407,1733407683113 2024-12-05T14:08:04,293 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T14:08:04,304 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da6aa2204f50%2C46407%2C1733407683113, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/WALs/da6aa2204f50,46407,1733407683113, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/oldWALs, maxLogs=10 2024-12-05T14:08:04,336 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/WALs/da6aa2204f50,46407,1733407683113/da6aa2204f50%2C46407%2C1733407683113.1733407684308 2024-12-05T14:08:04,337 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T14:08:04,337 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T14:08:04,338 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:08:04,341 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,341 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T14:08:04,405 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:04,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:04,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T14:08:04,412 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:04,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:08:04,413 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,416 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T14:08:04,417 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:04,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:08:04,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T14:08:04,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:04,422 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:08:04,422 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,426 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,426 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,429 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,430 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,432 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T14:08:04,434 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:08:04,440 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:08:04,441 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59593641, jitterRate=-0.11198554933071136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T14:08:04,449 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733407684353Initializing all the Stores at 1733407684353Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733407684354 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733407684354Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733407684355 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733407684355Cleaning up temporary data from old regions at 1733407684430 (+75 ms)Region opened successfully at 1733407684448 (+18 ms) 2024-12-05T14:08:04,451 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T14:08:04,480 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ba66f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da6aa2204f50/172.17.0.2:0 2024-12-05T14:08:04,512 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T14:08:04,524 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T14:08:04,524 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T14:08:04,527 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:08:04,528 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T14:08:04,535 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T14:08:04,536 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T14:08:04,563 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T14:08:04,574 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T14:08:04,582 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T14:08:04,585 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T14:08:04,586 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T14:08:04,649 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T14:08:04,653 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T14:08:04,659 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T14:08:04,666 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T14:08:04,668 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T14:08:04,674 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T14:08:04,696 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T14:08:04,707 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T14:08:04,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T14:08:04,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T14:08:04,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,728 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da6aa2204f50,46407,1733407683113, sessionid=0x101a71bd3a40000, setting cluster-up flag (Was=false) 2024-12-05T14:08:04,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,783 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T14:08:04,786 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da6aa2204f50,46407,1733407683113 2024-12-05T14:08:04,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:04,833 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T14:08:04,837 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da6aa2204f50,46407,1733407683113 2024-12-05T14:08:04,845 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T14:08:04,882 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(746): ClusterId : 7d1a65e7-77b1-485d-b2ac-464e96ef5ab4 2024-12-05T14:08:04,884 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T14:08:04,893 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T14:08:04,893 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T14:08:04,900 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T14:08:04,901 DEBUG [RS:0;da6aa2204f50:34743 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fcf2a32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da6aa2204f50/172.17.0.2:0 2024-12-05T14:08:04,913 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:08:04,918 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da6aa2204f50:34743 2024-12-05T14:08:04,922 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T14:08:04,923 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T14:08:04,923 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T14:08:04,926 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(2659): reportForDuty to master=da6aa2204f50,46407,1733407683113 with port=34743, startcode=1733407683864 2024-12-05T14:08:04,926 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T14:08:04,932 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T14:08:04,937 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da6aa2204f50,46407,1733407683113 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T14:08:04,940 DEBUG [RS:0;da6aa2204f50:34743 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T14:08:04,944 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da6aa2204f50:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:08:04,945 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da6aa2204f50:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:08:04,945 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da6aa2204f50:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:08:04,945 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da6aa2204f50:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:08:04,945 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da6aa2204f50:0, corePoolSize=10, maxPoolSize=10 2024-12-05T14:08:04,945 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:04,946 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da6aa2204f50:0, corePoolSize=2, maxPoolSize=2 2024-12-05T14:08:04,946 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:04,948 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733407714948 2024-12-05T14:08:04,950 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T14:08:04,950 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T14:08:04,951 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T14:08:04,951 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T14:08:04,953 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T14:08:04,954 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T14:08:04,954 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T14:08:04,954 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T14:08:04,956 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:04,955 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:04,956 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T14:08:04,958 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T14:08:04,959 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T14:08:04,959 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T14:08:04,963 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T14:08:04,963 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T14:08:04,965 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da6aa2204f50:0:becomeActiveMaster-HFileCleaner.large.0-1733407684964,5,FailOnTimeoutGroup] 2024-12-05T14:08:04,965 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da6aa2204f50:0:becomeActiveMaster-HFileCleaner.small.0-1733407684965,5,FailOnTimeoutGroup] 2024-12-05T14:08:04,965 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:04,966 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T14:08:04,966 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:04,967 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:04,975 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T14:08:04,976 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c 2024-12-05T14:08:04,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:08:04,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T14:08:04,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T14:08:04,992 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:04,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:04,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T14:08:04,996 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T14:08:04,996 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:04,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:04,997 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T14:08:05,000 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T14:08:05,000 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:05,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:05,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T14:08:05,004 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T14:08:05,004 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:05,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:05,005 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T14:08:05,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740 2024-12-05T14:08:05,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740 2024-12-05T14:08:05,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T14:08:05,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T14:08:05,007 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T14:08:05,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T14:08:05,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59509, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T14:08:05,015 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:08:05,015 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61176038, jitterRate=-0.08840599656105042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T14:08:05,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733407684988Initializing all the Stores at 1733407684988Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733407684988Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733407684988Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733407684988Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733407684988Cleaning up temporary data from old regions at 1733407685007 (+19 ms)Region opened successfully at 1733407685019 (+12 ms) 2024-12-05T14:08:05,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T14:08:05,020 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T14:08:05,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T14:08:05,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T14:08:05,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T14:08:05,020 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46407 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,022 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T14:08:05,022 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733407685019Disabling compacts and flushes for region at 1733407685019Disabling writes for close at 1733407685020 (+1 ms)Writing region close event to WAL at 1733407685021 (+1 ms)Closed at 1733407685021 2024-12-05T14:08:05,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46407 {}] master.ServerManager(517): Registering regionserver=da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,025 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T14:08:05,025 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T14:08:05,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T14:08:05,037 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c 2024-12-05T14:08:05,037 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T14:08:05,037 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T14:08:05,042 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:08:05,045 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T14:08:05,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T14:08:05,049 DEBUG [RS:0;da6aa2204f50:34743 {}] zookeeper.ZKUtil(111): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,050 WARN [RS:0;da6aa2204f50:34743 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T14:08:05,050 INFO [RS:0;da6aa2204f50:34743 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:08:05,050 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,052 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da6aa2204f50,34743,1733407683864] 2024-12-05T14:08:05,072 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T14:08:05,084 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T14:08:05,088 INFO [RS:0;da6aa2204f50:34743 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T14:08:05,088 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,089 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T14:08:05,093 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T14:08:05,094 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,094 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,095 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,095 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,095 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,095 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,095 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da6aa2204f50:0, corePoolSize=2, maxPoolSize=2 2024-12-05T14:08:05,095 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,095 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,096 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,096 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,096 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,096 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da6aa2204f50:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:08:05,096 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da6aa2204f50:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:08:05,096 DEBUG [RS:0;da6aa2204f50:34743 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da6aa2204f50:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:08:05,097 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,097 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,097 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,098 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,098 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,098 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,34743,1733407683864-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T14:08:05,115 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T14:08:05,116 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,34743,1733407683864-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,117 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,117 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.Replication(171): da6aa2204f50,34743,1733407683864 started 2024-12-05T14:08:05,132 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,132 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1482): Serving as da6aa2204f50,34743,1733407683864, RpcServer on da6aa2204f50/172.17.0.2:34743, sessionid=0x101a71bd3a40001 2024-12-05T14:08:05,133 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T14:08:05,133 DEBUG [RS:0;da6aa2204f50:34743 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,133 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da6aa2204f50,34743,1733407683864' 2024-12-05T14:08:05,133 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T14:08:05,134 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T14:08:05,135 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T14:08:05,135 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T14:08:05,135 DEBUG [RS:0;da6aa2204f50:34743 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,135 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da6aa2204f50,34743,1733407683864' 2024-12-05T14:08:05,135 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T14:08:05,136 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T14:08:05,136 DEBUG [RS:0;da6aa2204f50:34743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T14:08:05,136 INFO [RS:0;da6aa2204f50:34743 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T14:08:05,136 INFO [RS:0;da6aa2204f50:34743 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T14:08:05,196 WARN [da6aa2204f50:46407 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T14:08:05,245 INFO [RS:0;da6aa2204f50:34743 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T14:08:05,246 INFO [RS:0;da6aa2204f50:34743 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=da6aa2204f50%2C34743%2C1733407683864, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/oldWALs, maxLogs=55 2024-12-05T14:08:05,264 INFO [RS:0;da6aa2204f50:34743 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864/da6aa2204f50%2C34743%2C1733407683864.1733407685249 2024-12-05T14:08:05,264 DEBUG [RS:0;da6aa2204f50:34743 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T14:08:05,451 DEBUG [da6aa2204f50:46407 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T14:08:05,464 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,469 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da6aa2204f50,34743,1733407683864, state=OPENING 2024-12-05T14:08:05,507 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T14:08:05,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:05,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:05,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:08:05,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:08:05,518 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:08:05,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da6aa2204f50,34743,1733407683864}] 2024-12-05T14:08:05,691 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T14:08:05,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36675, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T14:08:05,703 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T14:08:05,703 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:08:05,704 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T14:08:05,705 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=da6aa2204f50%2C34743%2C1733407683864.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/oldWALs, maxLogs=55 2024-12-05T14:08:05,723 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864/da6aa2204f50%2C34743%2C1733407683864.meta.1733407685706.meta 2024-12-05T14:08:05,723 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T14:08:05,724 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T14:08:05,725 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T14:08:05,728 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T14:08:05,732 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T14:08:05,736 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T14:08:05,737 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:08:05,738 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T14:08:05,738 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T14:08:05,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T14:08:05,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T14:08:05,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:05,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:05,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T14:08:05,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T14:08:05,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:05,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:05,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T14:08:05,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T14:08:05,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:05,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:05,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T14:08:05,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T14:08:05,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:05,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:08:05,744 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T14:08:05,746 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740 2024-12-05T14:08:05,747 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740 2024-12-05T14:08:05,747 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T14:08:05,747 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T14:08:05,748 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T14:08:05,749 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T14:08:05,750 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64494100, jitterRate=-0.03896301984786987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T14:08:05,750 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T14:08:05,752 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733407685738Writing region info on filesystem at 1733407685739 (+1 ms)Initializing all the Stores at 1733407685739Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733407685739Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733407685739Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733407685739Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733407685739Cleaning up temporary data from old regions at 1733407685747 (+8 ms)Running coprocessor post-open hooks at 1733407685750 (+3 ms)Region opened successfully at 1733407685752 (+2 ms) 2024-12-05T14:08:05,757 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733407685682 2024-12-05T14:08:05,766 DEBUG [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T14:08:05,767 INFO [RS_OPEN_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T14:08:05,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,768 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da6aa2204f50,34743,1733407683864, state=OPEN 2024-12-05T14:08:05,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:08:05,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:08:05,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:08:05,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:08:05,808 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da6aa2204f50,34743,1733407683864 2024-12-05T14:08:05,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T14:08:05,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da6aa2204f50,34743,1733407683864 in 288 msec 2024-12-05T14:08:05,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T14:08:05,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 780 msec 2024-12-05T14:08:05,817 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T14:08:05,817 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T14:08:05,836 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T14:08:05,837 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da6aa2204f50,34743,1733407683864, seqNum=-1] 2024-12-05T14:08:05,853 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T14:08:05,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43105, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T14:08:05,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 997 msec 2024-12-05T14:08:05,870 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733407685870, completionTime=-1 2024-12-05T14:08:05,872 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T14:08:05,872 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:08:05,894 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T14:08:05,894 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733407745894 2024-12-05T14:08:05,894 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733407805894 2024-12-05T14:08:05,894 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-05T14:08:05,897 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,46407,1733407683113-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,897 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,46407,1733407683113-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,897 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,46407,1733407683113-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,898 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da6aa2204f50:46407, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,899 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,899 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,904 DEBUG [master/da6aa2204f50:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T14:08:05,925 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.866sec 2024-12-05T14:08:05,926 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T14:08:05,927 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T14:08:05,928 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T14:08:05,929 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T14:08:05,929 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T14:08:05,929 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,46407,1733407683113-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T14:08:05,930 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,46407,1733407683113-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T14:08:05,933 DEBUG [master/da6aa2204f50:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T14:08:05,934 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T14:08:05,935 INFO [master/da6aa2204f50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6aa2204f50,46407,1733407683113-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:08:05,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1b595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T14:08:05,998 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T14:08:05,998 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T14:08:06,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da6aa2204f50,46407,-1 for getting cluster id 2024-12-05T14:08:06,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T14:08:06,011 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7d1a65e7-77b1-485d-b2ac-464e96ef5ab4' 2024-12-05T14:08:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T14:08:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7d1a65e7-77b1-485d-b2ac-464e96ef5ab4" 2024-12-05T14:08:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de07bca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T14:08:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da6aa2204f50,46407,-1] 2024-12-05T14:08:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T14:08:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:08:06,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T14:08:06,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5a12a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T14:08:06,040 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T14:08:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da6aa2204f50,34743,1733407683864, seqNum=-1] 2024-12-05T14:08:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T14:08:06,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T14:08:06,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da6aa2204f50,46407,1733407683113 2024-12-05T14:08:06,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T14:08:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is da6aa2204f50,46407,1733407683113 2024-12-05T14:08:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b6bc802 2024-12-05T14:08:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T14:08:06,087 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T14:08:06,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46407 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T14:08:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46407 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-12-05T14:08:06,100 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T14:08:06,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46407 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-12-05T14:08:06,102 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:06,104 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T14:08:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46407 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:08:06,116 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0c4a5723b23bb0669905adeda91fc97d, NAME => 'table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c 2024-12-05T14:08:06,123 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:08:06,123 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 0c4a5723b23bb0669905adeda91fc97d, disabling compactions & flushes 2024-12-05T14:08:06,123 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:06,123 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:06,123 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. after waiting 0 ms 2024-12-05T14:08:06,123 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:06,123 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:06,123 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0c4a5723b23bb0669905adeda91fc97d: Waiting for close lock at 1733407686123Disabling compacts and flushes for region at 1733407686123Disabling writes for close at 1733407686123Writing region close event to WAL at 1733407686123Closed at 1733407686123 2024-12-05T14:08:06,125 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T14:08:06,129 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1733407686125"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733407686125"}]},"ts":"1733407686125"} 2024-12-05T14:08:06,134 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T14:08:06,135 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T14:08:06,138 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733407686135"}]},"ts":"1733407686135"} 2024-12-05T14:08:06,142 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-12-05T14:08:06,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=0c4a5723b23bb0669905adeda91fc97d, ASSIGN}] 2024-12-05T14:08:06,145 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=0c4a5723b23bb0669905adeda91fc97d, ASSIGN 2024-12-05T14:08:06,146 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=0c4a5723b23bb0669905adeda91fc97d, ASSIGN; state=OFFLINE, location=da6aa2204f50,34743,1733407683864; forceNewPlan=false, retain=false 2024-12-05T14:08:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46407 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:08:06,300 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0c4a5723b23bb0669905adeda91fc97d, regionState=OPENING, regionLocation=da6aa2204f50,34743,1733407683864 2024-12-05T14:08:06,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=0c4a5723b23bb0669905adeda91fc97d, ASSIGN because future has completed 2024-12-05T14:08:06,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c4a5723b23bb0669905adeda91fc97d, server=da6aa2204f50,34743,1733407683864}] 2024-12-05T14:08:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46407 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:08:06,467 INFO [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:06,468 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c4a5723b23bb0669905adeda91fc97d, NAME => 'table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d.', STARTKEY => '', ENDKEY => ''} 2024-12-05T14:08:06,469 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,469 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:08:06,469 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,469 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,471 INFO [StoreOpener-0c4a5723b23bb0669905adeda91fc97d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,476 INFO [StoreOpener-0c4a5723b23bb0669905adeda91fc97d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4a5723b23bb0669905adeda91fc97d columnFamilyName family 2024-12-05T14:08:06,477 DEBUG [StoreOpener-0c4a5723b23bb0669905adeda91fc97d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:08:06,477 INFO [StoreOpener-0c4a5723b23bb0669905adeda91fc97d-1 {}] regionserver.HStore(327): Store=0c4a5723b23bb0669905adeda91fc97d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:08:06,477 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,478 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,479 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,479 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,479 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,480 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,486 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:08:06,487 INFO [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0c4a5723b23bb0669905adeda91fc97d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60575139, jitterRate=-0.09736008942127228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:08:06,487 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:06,487 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0c4a5723b23bb0669905adeda91fc97d: Running coprocessor pre-open hook at 1733407686470Writing region info on filesystem at 1733407686470Initializing all the Stores at 1733407686470Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733407686470Cleaning up temporary data from old regions at 1733407686479 (+9 ms)Running coprocessor post-open hooks at 1733407686487 (+8 ms)Region opened successfully at 1733407686487 2024-12-05T14:08:06,488 INFO [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d., pid=6, masterSystemTime=1733407686463 2024-12-05T14:08:06,491 DEBUG [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:06,491 INFO [RS_OPEN_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:06,491 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0c4a5723b23bb0669905adeda91fc97d, regionState=OPEN, openSeqNum=2, regionLocation=da6aa2204f50,34743,1733407683864 2024-12-05T14:08:06,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c4a5723b23bb0669905adeda91fc97d, server=da6aa2204f50,34743,1733407683864 because future has completed 2024-12-05T14:08:06,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T14:08:06,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0c4a5723b23bb0669905adeda91fc97d, server=da6aa2204f50,34743,1733407683864 in 188 msec 2024-12-05T14:08:06,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T14:08:06,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=0c4a5723b23bb0669905adeda91fc97d, ASSIGN in 356 msec 2024-12-05T14:08:06,502 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T14:08:06,503 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733407686503"}]},"ts":"1733407686503"} 2024-12-05T14:08:06,505 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-12-05T14:08:06,506 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T14:08:06,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 412 msec 2024-12-05T14:08:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46407 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:08:06,740 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-12-05T14:08:06,740 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-12-05T14:08:06,746 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:08:06,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-12-05T14:08:06,751 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:08:06,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-12-05T14:08:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d., hostname=da6aa2204f50,34743,1733407683864, seqNum=2] 2024-12-05T14:08:06,773 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0c4a5723b23bb0669905adeda91fc97d 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-05T14:08:06,793 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6516d723c4444fa4b73df1b4fa172f22 2024-12-05T14:08:06,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/.tmp/family/6516d723c4444fa4b73df1b4fa172f22 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-05T14:08:06,827 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/.tmp/family/6516d723c4444fa4b73df1b4fa172f22 2024-12-05T14:08:06,856 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/.tmp/family/6516d723c4444fa4b73df1b4fa172f22 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22 2024-12-05T14:08:06,857 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22, delay=960 ms 2024-12-05T14:08:06,858 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T14:08:06,863 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 0c4a5723b23bb0669905adeda91fc97d in 91ms, sequenceid=6, compaction requested=false 2024-12-05T14:08:06,864 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-12-05T14:08:06,866 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0c4a5723b23bb0669905adeda91fc97d: 2024-12-05T14:08:07,820 DEBUG [hfile-prefetch-1733407686857 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6516d723c4444fa4b73df1b4fa172f22_0. 2024-12-05T14:08:07,824 DEBUG [hfile-prefetch-1733407686857 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6516d723c4444fa4b73df1b4fa172f22_117. 2024-12-05T14:08:07,824 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22, with total blocks 2, and data blocks 1 2024-12-05T14:08:07,824 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T14:08:07,825 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T14:08:07,825 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22. Will sleep 100ms and try the verification again. 2024-12-05T14:08:07,925 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22, with total blocks 2, and data blocks 1 2024-12-05T14:08:07,926 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-05T14:08:07,926 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2293): found block for file 6516d723c4444fa4b73df1b4fa172f22 in the backing map. Acquiring read lock for offset 0 2024-12-05T14:08:07,926 DEBUG [hfile-prefetch-1733407686857 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/family/6516d723c4444fa4b73df1b4fa172f22 has now been fully cached. 2024-12-05T14:08:07,927 DEBUG [hfile-prefetch-1733407686857 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6516d723c4444fa4b73df1b4fa172f22 2024-12-05T14:08:07,942 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T14:08:07,942 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T14:08:07,943 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T14:08:07,943 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-05T14:08:07,965 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-05T14:08:07,965 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733407687942 2024-12-05T14:08:07,965 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T14:08:08,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T14:08:08,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T14:08:08,878 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;da6aa2204f50:34743,5,FailOnTimeoutGroup] 2024-12-05T14:08:08,878 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da6aa2204f50,34743,1733407683864' ***** 2024-12-05T14:08:08,878 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-05T14:08:08,878 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-12-05T14:08:08,879 INFO [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T14:08:08,879 INFO [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/bucket.persistence 2024-12-05T14:08:08,879 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T14:08:08,879 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T14:08:08,879 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T14:08:08,879 INFO [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T14:08:08,879 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T14:08:08,889 DEBUG [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1487): Thread RS:0;da6aa2204f50:34743 started persisting bucket cache to file 2024-12-05T14:08:08,890 DEBUG [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T14:08:08,890 DEBUG [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-05T14:08:08,893 DEBUG [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-05T14:08:08,893 DEBUG [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733407688889 2024-12-05T14:08:08,893 DEBUG [RS:0;da6aa2204f50:34743 {}] bucket.BucketCache(1509): Thread RS:0;da6aa2204f50:34743 finished persisting bucket cache to file, renaming 2024-12-05T14:08:08,902 INFO [regionserver/da6aa2204f50:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:08:08,902 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T14:08:08,919 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T14:08:08,919 INFO [RS:0;da6aa2204f50:34743 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T14:08:08,919 INFO [RS:0;da6aa2204f50:34743 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T14:08:08,919 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(3091): Received CLOSE for 0c4a5723b23bb0669905adeda91fc97d 2024-12-05T14:08:08,920 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(959): stopping server da6aa2204f50,34743,1733407683864 2024-12-05T14:08:08,920 INFO [RS:0;da6aa2204f50:34743 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T14:08:08,920 INFO [RS:0;da6aa2204f50:34743 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da6aa2204f50:34743. 2024-12-05T14:08:08,920 INFO [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T14:08:08,920 DEBUG [RS:0;da6aa2204f50:34743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:08:08,921 DEBUG [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0c4a5723b23bb0669905adeda91fc97d, disabling compactions & flushes 2024-12-05T14:08:08,921 INFO [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:08,921 DEBUG [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:08,921 DEBUG [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. after waiting 0 ms 2024-12-05T14:08:08,921 DEBUG [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:08,926 DEBUG [RS:0;da6aa2204f50:34743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:08:08,926 DEBUG [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/default/table1/0c4a5723b23bb0669905adeda91fc97d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T14:08:08,926 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T14:08:08,926 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T14:08:08,926 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T14:08:08,927 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T14:08:08,927 INFO [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:08,927 DEBUG [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0c4a5723b23bb0669905adeda91fc97d: Waiting for close lock at 1733407688920Running coprocessor pre-close hooks at 1733407688921 (+1 ms)Disabling compacts and flushes for region at 1733407688921Disabling writes for close at 1733407688921Writing region close event to WAL at 1733407688922 (+1 ms)Running coprocessor post-close hooks at 1733407688926 (+4 ms)Closed at 1733407688927 (+1 ms) 2024-12-05T14:08:08,928 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T14:08:08,928 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T14:08:08,928 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T14:08:08,928 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T14:08:08,928 DEBUG [RS_CLOSE_REGION-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d. 2024-12-05T14:08:08,928 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T14:08:08,928 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T14:08:08,928 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T14:08:08,928 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T14:08:08,928 DEBUG [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T14:08:08,928 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-12-05T14:08:08,937 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eae381a9bcdf4c9ab4cf3ab421973862 2024-12-05T14:08:08,951 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/info/eae381a9bcdf4c9ab4cf3ab421973862 is 133, key is table1,,1733407686089.0c4a5723b23bb0669905adeda91fc97d./info:regioninfo/1733407686491/Put/seqid=0 2024-12-05T14:08:08,952 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/info/eae381a9bcdf4c9ab4cf3ab421973862 2024-12-05T14:08:08,960 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a4d82cd0c9c547ec8ac8fd030ab4025d 2024-12-05T14:08:08,970 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/ns/a4d82cd0c9c547ec8ac8fd030ab4025d is 43, key is default/ns:d/1733407685858/Put/seqid=0 2024-12-05T14:08:08,971 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/ns/a4d82cd0c9c547ec8ac8fd030ab4025d 2024-12-05T14:08:08,979 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3bfacd99fc714efb817e7a89335f6000 2024-12-05T14:08:08,989 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/table/3bfacd99fc714efb817e7a89335f6000 is 42, key is table1/table:state/1733407686503/Put/seqid=0 2024-12-05T14:08:08,990 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/table/3bfacd99fc714efb817e7a89335f6000 2024-12-05T14:08:08,992 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/info/eae381a9bcdf4c9ab4cf3ab421973862 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/info/eae381a9bcdf4c9ab4cf3ab421973862 2024-12-05T14:08:08,993 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/info/eae381a9bcdf4c9ab4cf3ab421973862, delay=908 ms 2024-12-05T14:08:08,994 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/info/eae381a9bcdf4c9ab4cf3ab421973862, entries=10, sequenceid=11, filesize=6.3 K 2024-12-05T14:08:08,995 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/ns/a4d82cd0c9c547ec8ac8fd030ab4025d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/ns/a4d82cd0c9c547ec8ac8fd030ab4025d 2024-12-05T14:08:08,996 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/ns/a4d82cd0c9c547ec8ac8fd030ab4025d, delay=979 ms 2024-12-05T14:08:08,997 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/ns/a4d82cd0c9c547ec8ac8fd030ab4025d, entries=2, sequenceid=11, filesize=5.0 K 2024-12-05T14:08:08,997 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/.tmp/table/3bfacd99fc714efb817e7a89335f6000 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/table/3bfacd99fc714efb817e7a89335f6000 2024-12-05T14:08:08,998 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/table/3bfacd99fc714efb817e7a89335f6000, delay=939 ms 2024-12-05T14:08:09,000 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/table/3bfacd99fc714efb817e7a89335f6000, entries=2, sequenceid=11, filesize=5.1 K 2024-12-05T14:08:09,000 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=11, compaction requested=false 2024-12-05T14:08:09,000 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T14:08:09,002 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/info/eae381a9bcdf4c9ab4cf3ab421973862 2024-12-05T14:08:09,002 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/ns/a4d82cd0c9c547ec8ac8fd030ab4025d 2024-12-05T14:08:09,003 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/table/3bfacd99fc714efb817e7a89335f6000 2024-12-05T14:08:09,006 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-05T14:08:09,007 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T14:08:09,007 INFO [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T14:08:09,007 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733407688928Running coprocessor pre-close hooks at 1733407688928Disabling compacts and flushes for region at 1733407688928Disabling writes for close at 1733407688928Obtaining lock to block concurrent updates at 1733407688928Preparing flush snapshotting stores in 1588230740 at 1733407688928Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1733407688929 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733407688929Flushing 1588230740/info: creating writer at 1733407688930 (+1 ms)Flushing 1588230740/info: appending metadata at 1733407688950 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733407688950Flushing 1588230740/ns: creating writer at 1733407688954 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733407688970 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733407688970Flushing 1588230740/table: creating writer at 1733407688973 (+3 ms)Flushing 1588230740/table: appending metadata at 1733407688989 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733407688989Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fd2b116: reopening flushed file at 1733407688992 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72576ff5: reopening flushed file at 1733407688995 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@570ec84a: reopening flushed file at 1733407688997 (+2 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=11, compaction requested=false at 1733407689000 (+3 ms)Writing region close event to WAL at 1733407689003 (+3 ms)Running coprocessor post-close hooks at 1733407689007 (+4 ms)Closed at 1733407689007 2024-12-05T14:08:09,007 DEBUG [RS_CLOSE_META-regionserver/da6aa2204f50:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T14:08:09,112 INFO [regionserver/da6aa2204f50:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T14:08:09,113 INFO [regionserver/da6aa2204f50:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T14:08:09,128 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(976): stopping server da6aa2204f50,34743,1733407683864; all regions closed. 2024-12-05T14:08:09,134 DEBUG [RS:0;da6aa2204f50:34743 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/oldWALs 2024-12-05T14:08:09,135 INFO [RS:0;da6aa2204f50:34743 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da6aa2204f50%2C34743%2C1733407683864.meta:.meta(num 1733407685706) 2024-12-05T14:08:09,138 DEBUG [RS:0;da6aa2204f50:34743 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/oldWALs 2024-12-05T14:08:09,138 INFO [RS:0;da6aa2204f50:34743 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da6aa2204f50%2C34743%2C1733407683864:(num 1733407685249) 2024-12-05T14:08:09,138 DEBUG [RS:0;da6aa2204f50:34743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:08:09,138 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:08:09,138 INFO [RS:0;da6aa2204f50:34743 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T14:08:09,138 INFO [RS:0;da6aa2204f50:34743 {}] hbase.ChoreService(370): Chore service for: regionserver/da6aa2204f50:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T14:08:09,139 INFO [RS:0;da6aa2204f50:34743 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T14:08:09,139 INFO [regionserver/da6aa2204f50:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T14:08:09,139 INFO [RS:0;da6aa2204f50:34743 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34743 2024-12-05T14:08:09,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da6aa2204f50,34743,1733407683864 2024-12-05T14:08:09,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T14:08:09,157 INFO [RS:0;da6aa2204f50:34743 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T14:08:09,158 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da6aa2204f50,34743,1733407683864] 2024-12-05T14:08:09,173 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da6aa2204f50,34743,1733407683864 already deleted, retry=false 2024-12-05T14:08:09,174 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of da6aa2204f50,34743,1733407683864 on da6aa2204f50,46407,1733407683113 2024-12-05T14:08:09,178 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure da6aa2204f50,34743,1733407683864, splitWal=true, meta=true 2024-12-05T14:08:09,180 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for da6aa2204f50,34743,1733407683864 (carryingMeta=true) da6aa2204f50,34743,1733407683864/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@19378dd1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-05T14:08:09,180 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure da6aa2204f50,34743,1733407683864, splitWal=true, meta=true 2024-12-05T14:08:09,182 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure da6aa2204f50,34743,1733407683864, splitWal=true, meta=true, isMeta: true 2024-12-05T14:08:09,183 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864-splitting 2024-12-05T14:08:09,183 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864-splitting dir is empty, no logs to split. 2024-12-05T14:08:09,183 INFO [PEWorker-1 {}] master.SplitWALManager(105): da6aa2204f50,34743,1733407683864 WAL count=0, meta=true 2024-12-05T14:08:09,186 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/WALs/da6aa2204f50,34743,1733407683864-splitting dir is empty, no logs to split. 2024-12-05T14:08:09,187 INFO [PEWorker-1 {}] master.SplitWALManager(105): da6aa2204f50,34743,1733407683864 WAL count=0, meta=true 2024-12-05T14:08:09,187 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if da6aa2204f50,34743,1733407683864 WAL splitting is done? wals=0, meta=true 2024-12-05T14:08:09,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T14:08:09,190 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:08:09,190 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:08:09,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T14:08:09,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34743-0x101a71bd3a40001, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T14:08:09,267 INFO [RS:0;da6aa2204f50:34743 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T14:08:09,267 INFO [RS:0;da6aa2204f50:34743 {}] regionserver.HRegionServer(1031): Exiting; stopping=da6aa2204f50,34743,1733407683864; zookeeper connection closed. 2024-12-05T14:08:09,268 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@43626c4e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@43626c4e 2024-12-05T14:08:09,341 WARN [da6aa2204f50:46407 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T14:08:09,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T14:08:09,880 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T14:08:09,880 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:08:09,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:08:09,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:08:09,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T14:08:09,883 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T14:08:09,883 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=365092990, stopped=false 2024-12-05T14:08:09,884 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da6aa2204f50,46407,1733407683113 2024-12-05T14:08:09,885 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'da6aa2204f50,46407,1733407683113' ***** 2024-12-05T14:08:09,885 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-12-05T14:08:09,885 INFO [M:0;da6aa2204f50:46407 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T14:08:09,886 INFO [M:0;da6aa2204f50:46407 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;da6aa2204f50:46407. 2024-12-05T14:08:09,886 DEBUG [M:0;da6aa2204f50:46407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:08:09,886 DEBUG [M:0;da6aa2204f50:46407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:08:09,886 INFO [M:0;da6aa2204f50:46407 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T14:08:09,887 DEBUG [M:0;da6aa2204f50:46407 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T14:08:09,887 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T14:08:09,887 DEBUG [M:0;da6aa2204f50:46407 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T14:08:09,887 DEBUG [master/da6aa2204f50:0:becomeActiveMaster-HFileCleaner.large.0-1733407684964 {}] cleaner.HFileCleaner(306): Exit Thread[master/da6aa2204f50:0:becomeActiveMaster-HFileCleaner.large.0-1733407684964,5,FailOnTimeoutGroup] 2024-12-05T14:08:09,887 DEBUG [master/da6aa2204f50:0:becomeActiveMaster-HFileCleaner.small.0-1733407684965 {}] cleaner.HFileCleaner(306): Exit Thread[master/da6aa2204f50:0:becomeActiveMaster-HFileCleaner.small.0-1733407684965,5,FailOnTimeoutGroup] 2024-12-05T14:08:09,888 INFO [M:0;da6aa2204f50:46407 {}] hbase.ChoreService(370): Chore service for: master/da6aa2204f50:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T14:08:09,889 INFO [M:0;da6aa2204f50:46407 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T14:08:09,889 DEBUG [M:0;da6aa2204f50:46407 {}] master.HMaster(1795): Stopping service threads 2024-12-05T14:08:09,889 INFO [M:0;da6aa2204f50:46407 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T14:08:09,889 INFO [M:0;da6aa2204f50:46407 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:08:09,890 INFO [M:0;da6aa2204f50:46407 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:08:09,890 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T14:08:09,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T14:08:09,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:08:09,924 DEBUG [M:0;da6aa2204f50:46407 {}] zookeeper.ZKUtil(347): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T14:08:09,924 WARN [M:0;da6aa2204f50:46407 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T14:08:09,925 INFO [M:0;da6aa2204f50:46407 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/.lastflushedseqids 2024-12-05T14:08:09,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T14:08:09,933 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T14:08:09,934 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:08:09,942 INFO [M:0;da6aa2204f50:46407 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:08:10,091 DEBUG [da6aa2204f50:46407 {}] assignment.AssignmentManager(2455): Stopped! Dropping assign of 1 queued regions. 2024-12-05T14:08:10,092 INFO [M:0;da6aa2204f50:46407 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T14:08:10,092 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T14:08:10,092 INFO [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:10,093 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:10,093 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T14:08:10,093 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:10,094 INFO [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.16 KB heapSize=30.77 KB 2024-12-05T14:08:10,111 DEBUG [M:0;da6aa2204f50:46407 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8c125b38f10d448590e97b6f84623b42 2024-12-05T14:08:10,124 DEBUG [M:0;da6aa2204f50:46407 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c125b38f10d448590e97b6f84623b42 is 82, key is hbase:meta,,1/info:regioninfo/1733407685767/Put/seqid=0 2024-12-05T14:08:10,125 INFO [M:0;da6aa2204f50:46407 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c125b38f10d448590e97b6f84623b42 2024-12-05T14:08:10,132 DEBUG [M:0;da6aa2204f50:46407 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed9b95d10c2a498785146a1a61596a04 2024-12-05T14:08:10,143 DEBUG [M:0;da6aa2204f50:46407 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed9b95d10c2a498785146a1a61596a04 is 705, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733407686508/Put/seqid=0 2024-12-05T14:08:10,144 INFO [M:0;da6aa2204f50:46407 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.55 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed9b95d10c2a498785146a1a61596a04 2024-12-05T14:08:10,150 DEBUG [M:0;da6aa2204f50:46407 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8718bc4a27584b0eabd4af88f220feff 2024-12-05T14:08:10,162 DEBUG [M:0;da6aa2204f50:46407 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8718bc4a27584b0eabd4af88f220feff is 58, key is da6aa2204f50,34743,1733407683864/rs:/1733407689181/DeleteFamily/seqid=0 2024-12-05T14:08:10,162 INFO [M:0;da6aa2204f50:46407 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8718bc4a27584b0eabd4af88f220feff 2024-12-05T14:08:10,164 INFO [M:0;da6aa2204f50:46407 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8718bc4a27584b0eabd4af88f220feff 2024-12-05T14:08:10,164 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c125b38f10d448590e97b6f84623b42 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c125b38f10d448590e97b6f84623b42 2024-12-05T14:08:10,166 INFO [M:0;da6aa2204f50:46407 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c125b38f10d448590e97b6f84623b42, entries=8, sequenceid=64, filesize=5.5 K 2024-12-05T14:08:10,166 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed9b95d10c2a498785146a1a61596a04 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed9b95d10c2a498785146a1a61596a04 2024-12-05T14:08:10,167 INFO [M:0;da6aa2204f50:46407 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed9b95d10c2a498785146a1a61596a04, entries=8, sequenceid=64, filesize=6.5 K 2024-12-05T14:08:10,168 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8718bc4a27584b0eabd4af88f220feff as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8718bc4a27584b0eabd4af88f220feff 2024-12-05T14:08:10,169 INFO [M:0;da6aa2204f50:46407 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8718bc4a27584b0eabd4af88f220feff 2024-12-05T14:08:10,169 INFO [M:0;da6aa2204f50:46407 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28eb72e9-9efb-d817-006a-8e9c27c5f6e4/f2f03ccd-d46e-783f-748c-8166e8b7e65c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8718bc4a27584b0eabd4af88f220feff, entries=1, sequenceid=64, filesize=5.2 K 2024-12-05T14:08:10,170 INFO [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.16 KB/24735, heapSize ~30.48 KB/31208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=64, compaction requested=false 2024-12-05T14:08:10,172 INFO [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:08:10,172 DEBUG [M:0;da6aa2204f50:46407 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733407690092Disabling compacts and flushes for region at 1733407690092Disabling writes for close at 1733407690093 (+1 ms)Obtaining lock to block concurrent updates at 1733407690094 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733407690094Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24735, getHeapSize=31448, getOffHeapSize=0, getCellsCount=76 at 1733407690095 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733407690096 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733407690096Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733407690124 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733407690124Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733407690127 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733407690142 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733407690142Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733407690145 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733407690161 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733407690161Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37d21461: reopening flushed file at 1733407690164 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3420b623: reopening flushed file at 1733407690166 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6024f62c: reopening flushed file at 1733407690167 (+1 ms)Finished flush of dataSize ~24.16 KB/24735, heapSize ~30.48 KB/31208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=64, compaction requested=false at 1733407690170 (+3 ms)Writing region close event to WAL at 1733407690172 (+2 ms)Closed at 1733407690172 2024-12-05T14:08:10,173 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T14:08:10,173 INFO [M:0;da6aa2204f50:46407 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T14:08:10,174 INFO [M:0;da6aa2204f50:46407 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46407 2024-12-05T14:08:10,174 INFO [M:0;da6aa2204f50:46407 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T14:08:10,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T14:08:10,282 INFO [M:0;da6aa2204f50:46407 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T14:08:10,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46407-0x101a71bd3a40000, quorum=127.0.0.1:57001, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T14:08:10,290 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T14:08:10,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T14:08:10,310 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:08:10,317 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/da6aa2204f50:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733407688993 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733407688999 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733407686857 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/da6aa2204f50:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733407688996 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/da6aa2204f50:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=8498 (was 8806)