2024-11-18 06:48:13,359 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 06:48:13,380 main DEBUG Took 0.019245 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 06:48:13,380 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 06:48:13,381 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 06:48:13,382 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 06:48:13,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,389 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 06:48:13,401 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,402 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,402 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,403 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,403 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,403 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,404 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,405 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,405 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,405 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,406 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,406 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,407 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,407 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,408 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,409 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,410 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,410 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 06:48:13,411 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,411 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 06:48:13,412 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 06:48:13,414 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 06:48:13,415 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 06:48:13,416 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 06:48:13,417 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 06:48:13,417 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 06:48:13,425 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 06:48:13,428 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 06:48:13,429 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 06:48:13,430 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 06:48:13,430 main DEBUG createAppenders(={Console}) 2024-11-18 06:48:13,431 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-18 06:48:13,431 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 06:48:13,431 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-18 06:48:13,432 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 06:48:13,432 main DEBUG OutputStream closed 2024-11-18 06:48:13,432 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 06:48:13,432 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 06:48:13,433 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-18 06:48:13,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 06:48:13,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 06:48:13,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 06:48:13,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 06:48:13,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 06:48:13,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 06:48:13,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 06:48:13,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 06:48:13,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 06:48:13,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 06:48:13,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 06:48:13,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 06:48:13,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 06:48:13,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 06:48:13,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 06:48:13,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 06:48:13,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 06:48:13,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 06:48:13,517 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 06:48:13,517 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-18 06:48:13,518 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 06:48:13,518 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-18T06:48:13,533 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-18 06:48:13,536 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 06:48:13,536 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T06:48:13,843 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17 2024-11-18T06:48:13,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1427, ProcessCount=11, AvailableMemoryMB=5801 2024-11-18T06:48:13,992 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T06:48:14,100 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:14,116 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/cluster_e3617b17-a3b1-cafd-790d-6c86c72c24bd, deleteOnExit=true 2024-11-18T06:48:14,204 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/cluster_e3617b17-a3b1-cafd-790d-6c86c72c24bd/zookeeper_0, clientPort=52139, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/cluster_e3617b17-a3b1-cafd-790d-6c86c72c24bd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/cluster_e3617b17-a3b1-cafd-790d-6c86c72c24bd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T06:48:14,216 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52139 2024-11-18T06:48:14,217 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:14,310 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e with version=8 2024-11-18T06:48:14,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/hbase-staging 2024-11-18T06:48:14,408 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T06:48:14,681 INFO [Time-limited test {}] client.ConnectionUtils(128): master/75ed3c3b0308:0 server-side Connection retries=45 2024-11-18T06:48:14,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:14,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:14,701 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T06:48:14,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:14,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T06:48:14,837 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T06:48:14,924 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T06:48:14,961 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T06:48:14,972 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T06:48:14,976 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T06:48:15,018 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20972 (auto-detected) 2024-11-18T06:48:15,019 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T06:48:15,044 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41365 2024-11-18T06:48:15,066 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41365 connecting to ZooKeeper ensemble=127.0.0.1:52139 2024-11-18T06:48:15,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413650x0, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T06:48:15,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41365-0x10129d556f00000 connected 2024-11-18T06:48:15,132 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:15,133 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:15,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:15,145 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e, hbase.cluster.distributed=false 2024-11-18T06:48:15,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T06:48:15,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41365 2024-11-18T06:48:15,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41365 2024-11-18T06:48:15,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41365 2024-11-18T06:48:15,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41365 2024-11-18T06:48:15,185 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41365 2024-11-18T06:48:15,288 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/75ed3c3b0308:0 server-side Connection retries=45 2024-11-18T06:48:15,290 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:15,290 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:15,291 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T06:48:15,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:15,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T06:48:15,295 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T06:48:15,298 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T06:48:15,299 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43355 2024-11-18T06:48:15,302 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43355 connecting to ZooKeeper ensemble=127.0.0.1:52139 2024-11-18T06:48:15,302 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:15,304 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:15,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433550x0, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T06:48:15,309 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43355-0x10129d556f00001 connected 2024-11-18T06:48:15,309 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:15,315 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T06:48:15,328 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.cache 2024-11-18T06:48:15,336 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T06:48:15,337 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T06:48:15,340 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T06:48:15,342 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T06:48:15,343 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T06:48:15,351 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T06:48:15,354 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T06:48:15,355 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.persistence is complete. 2024-11-18T06:48:15,363 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T06:48:15,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T06:48:15,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T06:48:15,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43355 2024-11-18T06:48:15,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43355 2024-11-18T06:48:15,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43355 2024-11-18T06:48:15,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43355 2024-11-18T06:48:15,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43355 2024-11-18T06:48:15,404 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/75ed3c3b0308:0 server-side Connection retries=45 2024-11-18T06:48:15,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:15,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:15,405 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T06:48:15,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:15,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T06:48:15,406 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T06:48:15,407 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T06:48:15,408 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45391 2024-11-18T06:48:15,411 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45391 connecting to ZooKeeper ensemble=127.0.0.1:52139 2024-11-18T06:48:15,412 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:15,413 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:15,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453910x0, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T06:48:15,420 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45391-0x10129d556f00002 connected 2024-11-18T06:48:15,421 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:15,422 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T06:48:15,425 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.cache 2024-11-18T06:48:15,425 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T06:48:15,425 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T06:48:15,426 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T06:48:15,428 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T06:48:15,429 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T06:48:15,429 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T06:48:15,429 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T06:48:15,430 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.persistence is complete. 2024-11-18T06:48:15,432 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T06:48:15,434 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T06:48:15,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T06:48:15,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45391 2024-11-18T06:48:15,441 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45391 2024-11-18T06:48:15,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45391 2024-11-18T06:48:15,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45391 2024-11-18T06:48:15,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45391 2024-11-18T06:48:15,464 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;75ed3c3b0308:41365 2024-11-18T06:48:15,468 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:15,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:15,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:15,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:15,482 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:15,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T06:48:15,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T06:48:15,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:15,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:15,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:15,507 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T06:48:15,509 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/75ed3c3b0308,41365,1731912494484 from backup master directory 2024-11-18T06:48:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:15,513 WARN [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T06:48:15,513 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:15,515 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T06:48:15,517 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T06:48:15,527 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/hbase.id] with ID: f074c0e6-ecab-45a3-bef3-f209b4a8e988 2024-11-18T06:48:15,527 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.tmp/hbase.id 2024-11-18T06:48:15,534 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/hbase.id] 2024-11-18T06:48:15,576 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:15,582 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T06:48:15,602 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-18T06:48:15,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:15,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:15,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:15,633 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T06:48:15,637 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T06:48:15,685 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T06:48:15,691 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:15,750 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store 2024-11-18T06:48:15,770 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T06:48:15,774 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:15,775 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T06:48:15,775 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:15,775 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:15,777 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T06:48:15,777 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:15,777 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:15,778 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731912495775Disabling compacts and flushes for region at 1731912495775Disabling writes for close at 1731912495777 (+2 ms)Writing region close event to WAL at 1731912495777Closed at 1731912495777 2024-11-18T06:48:15,778 WARN [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/.initializing 2024-11-18T06:48:15,778 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/WALs/75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:15,792 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T06:48:15,807 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=75ed3c3b0308%2C41365%2C1731912494484, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/WALs/75ed3c3b0308,41365,1731912494484, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/oldWALs, maxLogs=10 2024-11-18T06:48:15,860 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/WALs/75ed3c3b0308,41365,1731912494484/75ed3c3b0308%2C41365%2C1731912494484.1731912495813 2024-11-18T06:48:15,861 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:15,862 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:15,863 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:15,867 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,869 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,952 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T06:48:15,957 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:15,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:15,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T06:48:15,965 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:15,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:15,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T06:48:15,972 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:15,973 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:15,973 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T06:48:15,979 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:15,979 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:15,980 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,984 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,985 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,990 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,990 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:15,993 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T06:48:15,996 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:16,003 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T06:48:16,005 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70562205, jitterRate=0.05145879089832306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T06:48:16,012 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731912495886Initializing all the Stores at 1731912495886Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912495887 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912495890 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912495890Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912495890Cleaning up temporary data from old regions at 1731912495991 (+101 ms)Region opened successfully at 1731912496011 (+20 ms) 2024-11-18T06:48:16,013 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T06:48:16,046 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@907be14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=75ed3c3b0308/172.17.0.2:0 2024-11-18T06:48:16,084 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T06:48:16,097 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T06:48:16,098 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T06:48:16,101 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T06:48:16,101 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T06:48:16,109 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-18T06:48:16,109 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T06:48:16,143 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T06:48:16,154 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T06:48:16,157 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T06:48:16,159 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T06:48:16,161 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T06:48:16,162 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T06:48:16,165 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T06:48:16,168 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T06:48:16,170 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T06:48:16,171 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T06:48:16,173 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T06:48:16,192 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T06:48:16,194 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T06:48:16,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:16,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:16,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:16,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,210 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=75ed3c3b0308,41365,1731912494484, sessionid=0x10129d556f00000, setting cluster-up flag (Was=false) 2024-11-18T06:48:16,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,234 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T06:48:16,235 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:16,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,246 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T06:48:16,248 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:16,254 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T06:48:16,330 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:16,341 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T06:48:16,348 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T06:48:16,353 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(746): ClusterId : f074c0e6-ecab-45a3-bef3-f209b4a8e988 2024-11-18T06:48:16,353 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(746): ClusterId : f074c0e6-ecab-45a3-bef3-f209b4a8e988 2024-11-18T06:48:16,356 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T06:48:16,356 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T06:48:16,356 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 75ed3c3b0308,41365,1731912494484 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T06:48:16,364 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T06:48:16,364 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T06:48:16,364 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T06:48:16,364 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T06:48:16,366 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:16,366 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:16,366 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:16,366 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:16,367 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/75ed3c3b0308:0, corePoolSize=10, maxPoolSize=10 2024-11-18T06:48:16,367 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,367 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=2, maxPoolSize=2 2024-11-18T06:48:16,367 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,367 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T06:48:16,368 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T06:48:16,368 DEBUG [RS:1;75ed3c3b0308:45391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e64572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=75ed3c3b0308/172.17.0.2:0 2024-11-18T06:48:16,368 DEBUG [RS:0;75ed3c3b0308:43355 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed5e34a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=75ed3c3b0308/172.17.0.2:0 2024-11-18T06:48:16,372 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:16,373 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T06:48:16,376 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731912526376 2024-11-18T06:48:16,377 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:16,377 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T06:48:16,377 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T06:48:16,379 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T06:48:16,384 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T06:48:16,385 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T06:48:16,385 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T06:48:16,385 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T06:48:16,392 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,396 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;75ed3c3b0308:43355 2024-11-18T06:48:16,397 DEBUG [RS:1;75ed3c3b0308:45391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;75ed3c3b0308:45391 2024-11-18T06:48:16,400 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T06:48:16,400 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T06:48:16,400 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T06:48:16,400 DEBUG [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T06:48:16,400 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T06:48:16,400 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T06:48:16,401 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T06:48:16,403 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T06:48:16,403 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T06:48:16,403 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(2659): reportForDuty to master=75ed3c3b0308,41365,1731912494484 with port=43355, startcode=1731912495245 2024-11-18T06:48:16,403 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(2659): reportForDuty to master=75ed3c3b0308,41365,1731912494484 with port=45391, startcode=1731912495403 2024-11-18T06:48:16,408 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T06:48:16,409 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T06:48:16,414 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.large.0-1731912496410,5,FailOnTimeoutGroup] 2024-11-18T06:48:16,417 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.small.0-1731912496415,5,FailOnTimeoutGroup] 2024-11-18T06:48:16,417 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,418 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T06:48:16,418 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T06:48:16,418 DEBUG [RS:1;75ed3c3b0308:45391 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T06:48:16,418 DEBUG [RS:0;75ed3c3b0308:43355 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T06:48:16,419 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,419 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:16,420 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:16,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T06:48:16,439 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T06:48:16,439 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:16,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:16,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T06:48:16,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T06:48:16,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:16,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:16,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T06:48:16,449 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T06:48:16,449 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:16,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:16,450 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T06:48:16,454 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T06:48:16,454 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:16,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:16,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T06:48:16,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:16,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:16,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T06:48:16,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T06:48:16,457 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T06:48:16,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T06:48:16,476 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T06:48:16,477 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67125674, jitterRate=2.504885196685791E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T06:48:16,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731912496433Initializing all the Stores at 1731912496433Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912496433Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912496433Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912496433Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912496433Cleaning up temporary data from old regions at 1731912496456 (+23 ms)Region opened successfully at 1731912496481 (+25 ms) 2024-11-18T06:48:16,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T06:48:16,482 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T06:48:16,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T06:48:16,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T06:48:16,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T06:48:16,484 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:16,484 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731912496482Disabling compacts and flushes for region at 1731912496482Disabling writes for close at 1731912496482Writing region close event to WAL at 1731912496483 (+1 ms)Closed at 1731912496484 (+1 ms) 2024-11-18T06:48:16,487 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:16,487 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T06:48:16,496 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43433, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T06:48:16,496 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50423, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T06:48:16,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T06:48:16,504 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:16,505 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T06:48:16,506 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41365 {}] master.ServerManager(517): Registering regionserver=75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:16,507 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T06:48:16,516 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:16,517 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41365 {}] master.ServerManager(517): Registering regionserver=75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:16,520 DEBUG [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:16,520 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:16,520 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T06:48:16,520 DEBUG [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T06:48:16,520 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T06:48:16,520 DEBUG [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T06:48:16,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T06:48:16,530 DEBUG [RS:0;75ed3c3b0308:43355 {}] zookeeper.ZKUtil(111): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:16,530 DEBUG [RS:1;75ed3c3b0308:45391 {}] zookeeper.ZKUtil(111): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:16,530 WARN [RS:1;75ed3c3b0308:45391 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T06:48:16,530 WARN [RS:0;75ed3c3b0308:43355 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T06:48:16,531 INFO [RS:1;75ed3c3b0308:45391 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:16,531 INFO [RS:0;75ed3c3b0308:43355 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:16,531 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:16,531 DEBUG [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:16,532 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [75ed3c3b0308,45391,1731912495403] 2024-11-18T06:48:16,532 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [75ed3c3b0308,43355,1731912495245] 2024-11-18T06:48:16,558 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T06:48:16,558 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T06:48:16,572 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T06:48:16,572 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T06:48:16,578 INFO [RS:1;75ed3c3b0308:45391 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T06:48:16,578 INFO [RS:0;75ed3c3b0308:43355 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T06:48:16,578 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,578 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,580 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T06:48:16,580 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T06:48:16,585 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T06:48:16,585 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T06:48:16,587 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,587 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,587 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,587 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,587 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,587 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,587 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,587 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,587 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/75ed3c3b0308:0, corePoolSize=2, maxPoolSize=2 2024-11-18T06:48:16,588 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/75ed3c3b0308:0, corePoolSize=2, maxPoolSize=2 2024-11-18T06:48:16,588 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,588 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,589 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,589 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,589 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,589 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:16,589 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:16,589 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:16,589 DEBUG [RS:1;75ed3c3b0308:45391 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:16,589 DEBUG [RS:0;75ed3c3b0308:43355 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:16,593 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,593 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,593 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,593 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,594 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,594 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,45391,1731912495403-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T06:48:16,595 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,595 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,595 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,595 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,595 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,596 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,43355,1731912495245-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T06:48:16,616 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T06:48:16,616 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T06:48:16,619 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,43355,1731912495245-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,619 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,45391,1731912495403-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,619 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,619 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.Replication(171): 75ed3c3b0308,43355,1731912495245 started 2024-11-18T06:48:16,620 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,620 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.Replication(171): 75ed3c3b0308,45391,1731912495403 started 2024-11-18T06:48:16,639 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,640 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1482): Serving as 75ed3c3b0308,43355,1731912495245, RpcServer on 75ed3c3b0308/172.17.0.2:43355, sessionid=0x10129d556f00001 2024-11-18T06:48:16,641 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T06:48:16,641 DEBUG [RS:0;75ed3c3b0308:43355 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:16,641 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,43355,1731912495245' 2024-11-18T06:48:16,641 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T06:48:16,642 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T06:48:16,643 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T06:48:16,643 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T06:48:16,644 DEBUG [RS:0;75ed3c3b0308:43355 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:16,644 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,43355,1731912495245' 2024-11-18T06:48:16,644 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T06:48:16,645 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T06:48:16,645 DEBUG [RS:0;75ed3c3b0308:43355 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T06:48:16,645 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:16,646 INFO [RS:0;75ed3c3b0308:43355 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T06:48:16,646 INFO [RS:0;75ed3c3b0308:43355 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T06:48:16,646 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(1482): Serving as 75ed3c3b0308,45391,1731912495403, RpcServer on 75ed3c3b0308/172.17.0.2:45391, sessionid=0x10129d556f00002 2024-11-18T06:48:16,646 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T06:48:16,646 DEBUG [RS:1;75ed3c3b0308:45391 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:16,646 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,45391,1731912495403' 2024-11-18T06:48:16,646 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T06:48:16,647 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T06:48:16,648 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T06:48:16,648 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T06:48:16,648 DEBUG [RS:1;75ed3c3b0308:45391 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:16,648 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,45391,1731912495403' 2024-11-18T06:48:16,648 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T06:48:16,649 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T06:48:16,650 DEBUG [RS:1;75ed3c3b0308:45391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T06:48:16,650 INFO [RS:1;75ed3c3b0308:45391 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T06:48:16,650 INFO [RS:1;75ed3c3b0308:45391 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T06:48:16,658 WARN [75ed3c3b0308:41365 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T06:48:16,751 INFO [RS:1;75ed3c3b0308:45391 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T06:48:16,751 INFO [RS:0;75ed3c3b0308:43355 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T06:48:16,752 INFO [RS:0;75ed3c3b0308:43355 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C43355%2C1731912495245, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,43355,1731912495245, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:16,752 INFO [RS:1;75ed3c3b0308:45391 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C45391%2C1731912495403, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,45391,1731912495403, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:16,777 INFO [RS:1;75ed3c3b0308:45391 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,45391,1731912495403/75ed3c3b0308%2C45391%2C1731912495403.1731912496757 2024-11-18T06:48:16,777 INFO [RS:0;75ed3c3b0308:43355 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,43355,1731912495245/75ed3c3b0308%2C43355%2C1731912495245.1731912496757 2024-11-18T06:48:16,777 DEBUG [RS:1;75ed3c3b0308:45391 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:16,777 DEBUG [RS:0;75ed3c3b0308:43355 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:16,910 DEBUG [75ed3c3b0308:41365 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T06:48:16,922 DEBUG [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(204): Hosts are {75ed3c3b0308=0} racks are {/default-rack=0} 2024-11-18T06:48:16,925 DEBUG [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T06:48:16,925 DEBUG [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T06:48:16,925 DEBUG [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T06:48:16,925 DEBUG [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T06:48:16,925 INFO [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T06:48:16,925 INFO [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T06:48:16,925 DEBUG [75ed3c3b0308:41365 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T06:48:16,932 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:16,938 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 75ed3c3b0308,43355,1731912495245, state=OPENING 2024-11-18T06:48:16,944 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T06:48:16,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:16,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:16,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:16,947 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:16,948 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T06:48:16,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=75ed3c3b0308,43355,1731912495245}] 2024-11-18T06:48:17,125 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T06:48:17,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50233, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T06:48:17,136 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T06:48:17,137 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:17,137 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T06:48:17,139 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C43355%2C1731912495245.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,43355,1731912495245, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:17,159 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,43355,1731912495245/75ed3c3b0308%2C43355%2C1731912495245.meta.1731912497140.meta 2024-11-18T06:48:17,159 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:17,160 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:17,161 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T06:48:17,164 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T06:48:17,168 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T06:48:17,172 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T06:48:17,173 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:17,173 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T06:48:17,173 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T06:48:17,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T06:48:17,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T06:48:17,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:17,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:17,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T06:48:17,177 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T06:48:17,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:17,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:17,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T06:48:17,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T06:48:17,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:17,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:17,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T06:48:17,180 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T06:48:17,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:17,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:17,180 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T06:48:17,181 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:17,182 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:17,182 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T06:48:17,182 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T06:48:17,182 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T06:48:17,184 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T06:48:17,184 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61967655, jitterRate=-0.07660998404026031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T06:48:17,184 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T06:48:17,186 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731912497174Writing region info on filesystem at 1731912497174Initializing all the Stores at 1731912497174Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912497174Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912497175 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912497175Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912497175Cleaning up temporary data from old regions at 1731912497182 (+7 ms)Running coprocessor post-open hooks at 1731912497184 (+2 ms)Region opened successfully at 1731912497186 (+2 ms) 2024-11-18T06:48:17,191 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731912497117 2024-11-18T06:48:17,202 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T06:48:17,202 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T06:48:17,203 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:17,205 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 75ed3c3b0308,43355,1731912495245, state=OPEN 2024-11-18T06:48:17,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:17,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:17,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:17,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:17,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:17,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:17,208 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:17,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T06:48:17,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=75ed3c3b0308,43355,1731912495245 in 258 msec 2024-11-18T06:48:17,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T06:48:17,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 716 msec 2024-11-18T06:48:17,217 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:17,217 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T06:48:17,238 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T06:48:17,239 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,43355,1731912495245, seqNum=-1] 2024-11-18T06:48:17,258 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T06:48:17,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T06:48:17,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 993 msec 2024-11-18T06:48:17,279 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731912497279, completionTime=-1 2024-11-18T06:48:17,282 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T06:48:17,282 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T06:48:17,309 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T06:48:17,309 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731912557309 2024-11-18T06:48:17,309 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731912617309 2024-11-18T06:48:17,309 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-18T06:48:17,319 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-18T06:48:17,326 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,41365,1731912494484-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:17,326 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,41365,1731912494484-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:17,326 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,41365,1731912494484-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:17,327 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-75ed3c3b0308:41365, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:17,328 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:17,328 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:17,333 DEBUG [master/75ed3c3b0308:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T06:48:17,354 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.841sec 2024-11-18T06:48:17,355 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T06:48:17,357 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T06:48:17,358 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T06:48:17,358 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T06:48:17,358 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T06:48:17,359 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,41365,1731912494484-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T06:48:17,359 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,41365,1731912494484-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T06:48:17,360 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T06:48:17,361 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T06:48:17,362 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,41365,1731912494484-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:17,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T06:48:17,465 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T06:48:17,465 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T06:48:17,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 75ed3c3b0308,41365,-1 for getting cluster id 2024-11-18T06:48:17,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T06:48:17,480 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f074c0e6-ecab-45a3-bef3-f209b4a8e988' 2024-11-18T06:48:17,483 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T06:48:17,483 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f074c0e6-ecab-45a3-bef3-f209b4a8e988" 2024-11-18T06:48:17,483 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160c0505, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T06:48:17,483 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [75ed3c3b0308,41365,-1] 2024-11-18T06:48:17,487 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T06:48:17,488 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:17,490 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T06:48:17,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T06:48:17,493 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T06:48:17,499 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,43355,1731912495245, seqNum=-1] 2024-11-18T06:48:17,500 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T06:48:17,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T06:48:17,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:17,542 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T06:48:17,548 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:17,550 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ffafc50 2024-11-18T06:48:17,551 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T06:48:17,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T06:48:17,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731912497536', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T06:48:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 2024-11-18T06:48:17,570 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T06:48:17,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731912497536" procId is: 4 2024-11-18T06:48:17,573 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:17,575 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T06:48:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:17,592 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d49f32e8a7e9a24677dcc914e95eb180, NAME => 'testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731912497536', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:17,600 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:17,600 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(1722): Closing d49f32e8a7e9a24677dcc914e95eb180, disabling compactions & flushes 2024-11-18T06:48:17,600 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:17,600 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:17,601 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. after waiting 0 ms 2024-11-18T06:48:17,601 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:17,601 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:17,601 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731912497536-pool-0 {}] regionserver.HRegion(1676): Region close journal for d49f32e8a7e9a24677dcc914e95eb180: Waiting for close lock at 1731912497600Disabling compacts and flushes for region at 1731912497600Disabling writes for close at 1731912497601 (+1 ms)Writing region close event to WAL at 1731912497601Closed at 1731912497601 2024-11-18T06:48:17,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T06:48:17,607 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731912497602"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731912497602"}]},"ts":"1731912497602"} 2024-11-18T06:48:17,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T06:48:17,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T06:48:17,615 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731912497536","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731912497613"}]},"ts":"1731912497613"} 2024-11-18T06:48:17,619 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731912497536, state=ENABLING in hbase:meta 2024-11-18T06:48:17,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {75ed3c3b0308=0} racks are {/default-rack=0} 2024-11-18T06:48:17,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T06:48:17,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T06:48:17,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T06:48:17,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T06:48:17,620 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T06:48:17,620 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T06:48:17,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T06:48:17,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, ASSIGN}] 2024-11-18T06:48:17,623 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, ASSIGN 2024-11-18T06:48:17,624 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, ASSIGN; state=OFFLINE, location=75ed3c3b0308,45391,1731912495403; forceNewPlan=false, retain=false 2024-11-18T06:48:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:17,777 INFO [75ed3c3b0308:41365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T06:48:17,778 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d49f32e8a7e9a24677dcc914e95eb180, regionState=OPENING, regionLocation=75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:17,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, ASSIGN because future has completed 2024-11-18T06:48:17,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,45391,1731912495403}] 2024-11-18T06:48:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:17,936 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T06:48:17,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49209, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T06:48:17,940 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:17,941 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d49f32e8a7e9a24677dcc914e95eb180, NAME => 'testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:17,941 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731912497536 d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,941 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:17,941 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,942 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,943 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,947 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49f32e8a7e9a24677dcc914e95eb180 columnFamilyName family 2024-11-18T06:48:17,948 DEBUG [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:17,948 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.HStore(327): Store=d49f32e8a7e9a24677dcc914e95eb180/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:17,949 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,949 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,949 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,950 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,950 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,950 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,957 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T06:48:17,958 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d49f32e8a7e9a24677dcc914e95eb180; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68619674, jitterRate=0.022512823343276978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T06:48:17,958 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:17,959 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d49f32e8a7e9a24677dcc914e95eb180: Running coprocessor pre-open hook at 1731912497942Writing region info on filesystem at 1731912497942Initializing all the Stores at 1731912497942Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912497942Cleaning up temporary data from old regions at 1731912497950 (+8 ms)Running coprocessor post-open hooks at 1731912497958 (+8 ms)Region opened successfully at 1731912497959 (+1 ms) 2024-11-18T06:48:17,960 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180., pid=6, masterSystemTime=1731912497935 2024-11-18T06:48:17,964 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:17,964 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:17,964 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d49f32e8a7e9a24677dcc914e95eb180, regionState=OPEN, openSeqNum=2, regionLocation=75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:17,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,45391,1731912495403 because future has completed 2024-11-18T06:48:17,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T06:48:17,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,45391,1731912495403 in 187 msec 2024-11-18T06:48:17,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T06:48:17,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, ASSIGN in 350 msec 2024-11-18T06:48:17,976 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T06:48:17,976 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731912497536","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731912497976"}]},"ts":"1731912497976"} 2024-11-18T06:48:17,979 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731912497536, state=ENABLED in hbase:meta 2024-11-18T06:48:17,980 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T06:48:17,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731912497536 in 416 msec 2024-11-18T06:48:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:18,210 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731912497536 get assigned. Timeout = 60000ms 2024-11-18T06:48:18,210 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731912497536 completed 2024-11-18T06:48:18,213 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T06:48:18,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731912497536 assigned to meta. Checking AM states. 2024-11-18T06:48:18,219 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T06:48:18,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731912497536 assigned. 2024-11-18T06:48:18,229 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731912497536', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180., hostname=75ed3c3b0308,45391,1731912495403, seqNum=2] 2024-11-18T06:48:18,231 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T06:48:18,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T06:48:18,242 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d49f32e8a7e9a24677dcc914e95eb180 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T06:48:18,268 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07b60d646baf481ca8968a204718d584 2024-11-18T06:48:18,305 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/.tmp/family/07b60d646baf481ca8968a204718d584 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T06:48:18,310 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/.tmp/family/07b60d646baf481ca8968a204718d584 2024-11-18T06:48:18,347 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/.tmp/family/07b60d646baf481ca8968a204718d584 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584 2024-11-18T06:48:18,349 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584, delay=978 ms 2024-11-18T06:48:18,350 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T06:48:18,356 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d49f32e8a7e9a24677dcc914e95eb180 in 113ms, sequenceid=6, compaction requested=false 2024-11-18T06:48:18,358 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731912497536' 2024-11-18T06:48:18,359 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d49f32e8a7e9a24677dcc914e95eb180: 2024-11-18T06:48:19,329 DEBUG [hfile-prefetch-1731912498349 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 07b60d646baf481ca8968a204718d584_0. 2024-11-18T06:48:19,333 DEBUG [hfile-prefetch-1731912498349 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 07b60d646baf481ca8968a204718d584_117. 2024-11-18T06:48:19,334 DEBUG [hfile-prefetch-1731912498349 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584, with total blocks 2, and data blocks 1 2024-11-18T06:48:19,334 DEBUG [hfile-prefetch-1731912498349 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T06:48:19,334 DEBUG [hfile-prefetch-1731912498349 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T06:48:19,334 DEBUG [hfile-prefetch-1731912498349 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584. Will sleep 100ms and try the verification again. 2024-11-18T06:48:19,343 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T06:48:19,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T06:48:19,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:19,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:19,360 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T06:48:19,361 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731912499343 2024-11-18T06:48:19,361 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T06:48:19,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=d49f32e8a7e9a24677dcc914e95eb180, source=75ed3c3b0308,45391,1731912495403, destination=75ed3c3b0308,43355,1731912495245, warming up region on 75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:19,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=d49f32e8a7e9a24677dcc914e95eb180, source=75ed3c3b0308,45391,1731912495403, destination=75ed3c3b0308,43355,1731912495245, running balancer 2024-11-18T06:48:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, REOPEN/MOVE 2024-11-18T06:48:19,372 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, REOPEN/MOVE 2024-11-18T06:48:19,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(7855): Warmup {ENCODED => d49f32e8a7e9a24677dcc914e95eb180, NAME => 'testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:19,374 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,374 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d49f32e8a7e9a24677dcc914e95eb180, regionState=CLOSING, regionLocation=75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:19,375 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49f32e8a7e9a24677dcc914e95eb180 columnFamilyName family 2024-11-18T06:48:19,375 DEBUG [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:19,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, REOPEN/MOVE because future has completed 2024-11-18T06:48:19,377 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-18T06:48:19,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,45391,1731912495403}] 2024-11-18T06:48:19,379 DEBUG [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584 2024-11-18T06:48:19,379 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.HStore(327): Store=d49f32e8a7e9a24677dcc914e95eb180/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(1722): Closing d49f32e8a7e9a24677dcc914e95eb180, disabling compactions & flushes 2024-11-18T06:48:19,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. after waiting 0 ms 2024-11-18T06:48:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,380 DEBUG [StoreFileCloser-d49f32e8a7e9a24677dcc914e95eb180-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584 2024-11-18T06:48:19,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] regionserver.HRegion(1676): Region close journal for d49f32e8a7e9a24677dcc914e95eb180: Waiting for close lock at 1731912499379Disabling compacts and flushes for region at 1731912499379Disabling writes for close at 1731912499379Writing region close event to WAL at 1731912499381 (+2 ms)Closed at 1731912499381 2024-11-18T06:48:19,383 WARN [hfile-prefetch-1731912498349 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T06:48:19,384 DEBUG [hfile-prefetch-1731912498349 {}] hfile.PrefetchExecutor(121): Prefetch completed for 07b60d646baf481ca8968a204718d584 2024-11-18T06:48:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-18T06:48:19,535 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,535 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-18T06:48:19,536 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing d49f32e8a7e9a24677dcc914e95eb180, disabling compactions & flushes 2024-11-18T06:48:19,536 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,536 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,536 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. after waiting 0 ms 2024-11-18T06:48:19,536 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,538 DEBUG [StoreFileCloser-d49f32e8a7e9a24677dcc914e95eb180-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 07b60d646baf481ca8968a204718d584, starting offset: 0, end offset: 9223372036854775807 2024-11-18T06:48:19,539 DEBUG [StoreFileCloser-d49f32e8a7e9a24677dcc914e95eb180-family-1 {}] bucket.BucketCache(856): removed key 07b60d646baf481ca8968a204718d584_0 from back map with offset lock 2094080 in the evict process 2024-11-18T06:48:19,544 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T06:48:19,545 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,545 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for d49f32e8a7e9a24677dcc914e95eb180: Waiting for close lock at 1731912499536Running coprocessor pre-close hooks at 1731912499536Disabling compacts and flushes for region at 1731912499536Disabling writes for close at 1731912499536Writing region close event to WAL at 1731912499540 (+4 ms)Running coprocessor post-close hooks at 1731912499545 (+5 ms)Closed at 1731912499545 2024-11-18T06:48:19,545 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding d49f32e8a7e9a24677dcc914e95eb180 move to 75ed3c3b0308,43355,1731912495245 record at close sequenceid=6 2024-11-18T06:48:19,548 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,549 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d49f32e8a7e9a24677dcc914e95eb180, regionState=CLOSED 2024-11-18T06:48:19,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,45391,1731912495403 because future has completed 2024-11-18T06:48:19,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T06:48:19,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,45391,1731912495403 in 174 msec 2024-11-18T06:48:19,555 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, REOPEN/MOVE; state=CLOSED, location=75ed3c3b0308,43355,1731912495245; forceNewPlan=false, retain=false 2024-11-18T06:48:19,561 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T06:48:19,561 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T06:48:19,561 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:19,561 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:19,564 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T06:48:19,564 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731912499561 2024-11-18T06:48:19,564 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T06:48:19,705 INFO [75ed3c3b0308:41365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T06:48:19,706 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d49f32e8a7e9a24677dcc914e95eb180, regionState=OPENING, regionLocation=75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:19,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, REOPEN/MOVE because future has completed 2024-11-18T06:48:19,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,43355,1731912495245}] 2024-11-18T06:48:19,864 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,864 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => d49f32e8a7e9a24677dcc914e95eb180, NAME => 'testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:19,865 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731912497536 d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,865 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:19,865 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,865 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,866 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,867 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49f32e8a7e9a24677dcc914e95eb180 columnFamilyName family 2024-11-18T06:48:19,867 DEBUG [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:19,870 DEBUG [StoreFileOpener-d49f32e8a7e9a24677dcc914e95eb180-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584, delay=933 ms 2024-11-18T06:48:19,871 DEBUG [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584 2024-11-18T06:48:19,871 INFO [StoreOpener-d49f32e8a7e9a24677dcc914e95eb180-1 {}] regionserver.HStore(327): Store=d49f32e8a7e9a24677dcc914e95eb180/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:19,871 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,872 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,873 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,873 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,873 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,878 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,878 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened d49f32e8a7e9a24677dcc914e95eb180; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69143178, jitterRate=0.030313640832901}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T06:48:19,878 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,880 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for d49f32e8a7e9a24677dcc914e95eb180: Running coprocessor pre-open hook at 1731912499865Writing region info on filesystem at 1731912499865Initializing all the Stores at 1731912499866 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912499866Cleaning up temporary data from old regions at 1731912499873 (+7 ms)Running coprocessor post-open hooks at 1731912499878 (+5 ms)Region opened successfully at 1731912499880 (+2 ms) 2024-11-18T06:48:19,881 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180., pid=9, masterSystemTime=1731912499861 2024-11-18T06:48:19,883 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,883 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,884 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d49f32e8a7e9a24677dcc914e95eb180, regionState=OPEN, openSeqNum=10, regionLocation=75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:19,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,43355,1731912495245 because future has completed 2024-11-18T06:48:19,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-18T06:48:19,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure d49f32e8a7e9a24677dcc914e95eb180, server=75ed3c3b0308,43355,1731912495245 in 178 msec 2024-11-18T06:48:19,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731912497536, region=d49f32e8a7e9a24677dcc914e95eb180, REOPEN/MOVE in 518 msec 2024-11-18T06:48:19,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T06:48:19,925 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T06:48:19,925 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:19,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:19,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:19,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T06:48:19,931 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T06:48:19,931 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-18T06:48:19,931 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=75ed3c3b0308,41365,1731912494484 2024-11-18T06:48:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:19,933 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T06:48:19,934 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T06:48:19,934 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:19,934 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:19,934 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:19,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:19,935 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:19,935 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '75ed3c3b0308,43355,1731912495245' ***** 2024-11-18T06:48:19,935 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T06:48:19,935 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '75ed3c3b0308,45391,1731912495403' ***** 2024-11-18T06:48:19,935 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T06:48:19,935 INFO [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T06:48:19,935 INFO [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T06:48:19,936 INFO [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.persistence 2024-11-18T06:48:19,936 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T06:48:19,936 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T06:48:19,936 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T06:48:19,936 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T06:48:19,936 INFO [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/bucket.persistence 2024-11-18T06:48:19,936 INFO [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T06:48:19,936 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T06:48:19,936 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T06:48:19,937 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T06:48:19,937 INFO [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T06:48:19,937 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T06:48:19,946 DEBUG [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1487): Thread RS:0;75ed3c3b0308:43355 started persisting bucket cache to file 2024-11-18T06:48:19,947 DEBUG [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:19,947 DEBUG [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:19,947 DEBUG [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1487): Thread RS:1;75ed3c3b0308:45391 started persisting bucket cache to file 2024-11-18T06:48:19,947 DEBUG [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:19,947 DEBUG [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:19,951 DEBUG [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T06:48:19,951 DEBUG [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T06:48:19,951 DEBUG [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731912499947 2024-11-18T06:48:19,951 DEBUG [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731912499946 2024-11-18T06:48:19,951 DEBUG [RS:1;75ed3c3b0308:45391 {}] bucket.BucketCache(1509): Thread RS:1;75ed3c3b0308:45391 finished persisting bucket cache to file, renaming 2024-11-18T06:48:19,951 DEBUG [RS:0;75ed3c3b0308:43355 {}] bucket.BucketCache(1509): Thread RS:0;75ed3c3b0308:43355 finished persisting bucket cache to file, renaming 2024-11-18T06:48:19,951 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T06:48:19,951 INFO [RS:1;75ed3c3b0308:45391 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T06:48:19,951 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T06:48:19,952 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T06:48:19,952 INFO [RS:1;75ed3c3b0308:45391 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T06:48:19,952 INFO [RS:0;75ed3c3b0308:43355 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T06:48:19,952 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T06:48:19,952 INFO [RS:0;75ed3c3b0308:43355 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T06:48:19,952 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(959): stopping server 75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:19,952 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(3091): Received CLOSE for d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,952 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T06:48:19,952 INFO [RS:1;75ed3c3b0308:45391 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;75ed3c3b0308:45391. 2024-11-18T06:48:19,952 DEBUG [RS:1;75ed3c3b0308:45391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:19,952 DEBUG [RS:1;75ed3c3b0308:45391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:19,953 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(976): stopping server 75ed3c3b0308,45391,1731912495403; all regions closed. 2024-11-18T06:48:19,953 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(959): stopping server 75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:19,953 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T06:48:19,953 INFO [RS:0;75ed3c3b0308:43355 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;75ed3c3b0308:43355. 2024-11-18T06:48:19,953 DEBUG [RS:0;75ed3c3b0308:43355 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:19,953 DEBUG [RS:0;75ed3c3b0308:43355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:19,953 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T06:48:19,953 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T06:48:19,953 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T06:48:19,953 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T06:48:19,953 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T06:48:19,953 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d49f32e8a7e9a24677dcc914e95eb180, disabling compactions & flushes 2024-11-18T06:48:19,954 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,954 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T06:48:19,954 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,954 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. after waiting 0 ms 2024-11-18T06:48:19,954 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d49f32e8a7e9a24677dcc914e95eb180=testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180.} 2024-11-18T06:48:19,954 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T06:48:19,954 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,954 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T06:48:19,954 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T06:48:19,954 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T06:48:19,954 DEBUG [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d49f32e8a7e9a24677dcc914e95eb180 2024-11-18T06:48:19,954 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T06:48:19,954 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T06:48:19,954 DEBUG [StoreFileCloser-d49f32e8a7e9a24677dcc914e95eb180-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/family/07b60d646baf481ca8968a204718d584 2024-11-18T06:48:19,955 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-18T06:48:19,958 DEBUG [RS:1;75ed3c3b0308:45391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:19,958 INFO [RS:1;75ed3c3b0308:45391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C45391%2C1731912495403:(num 1731912496757) 2024-11-18T06:48:19,958 DEBUG [RS:1;75ed3c3b0308:45391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:19,958 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:19,958 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T06:48:19,959 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.ChoreService(370): Chore service for: regionserver/75ed3c3b0308:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T06:48:19,959 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T06:48:19,959 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T06:48:19,959 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T06:48:19,959 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T06:48:19,960 INFO [RS:1;75ed3c3b0308:45391 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45391 2024-11-18T06:48:19,961 INFO [regionserver/75ed3c3b0308:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T06:48:19,963 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnRegionMove1731912497536/d49f32e8a7e9a24677dcc914e95eb180/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T06:48:19,963 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,964 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d49f32e8a7e9a24677dcc914e95eb180: Waiting for close lock at 1731912499953Running coprocessor pre-close hooks at 1731912499953Disabling compacts and flushes for region at 1731912499953Disabling writes for close at 1731912499954 (+1 ms)Writing region close event to WAL at 1731912499955 (+1 ms)Running coprocessor post-close hooks at 1731912499963 (+8 ms)Closed at 1731912499963 2024-11-18T06:48:19,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T06:48:19,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/75ed3c3b0308,45391,1731912495403 2024-11-18T06:48:19,964 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T06:48:19,964 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180. 2024-11-18T06:48:19,967 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [75ed3c3b0308,45391,1731912495403] 2024-11-18T06:48:19,968 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/75ed3c3b0308,45391,1731912495403 already deleted, retry=false 2024-11-18T06:48:19,969 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 75ed3c3b0308,45391,1731912495403 expired; onlineServers=1 2024-11-18T06:48:19,969 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a1458fd8cbb3435b8d5780329ab9ae5c 2024-11-18T06:48:19,987 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/a1458fd8cbb3435b8d5780329ab9ae5c is 205, key is testBlockEvictionOnRegionMove1731912497536,,1731912497555.d49f32e8a7e9a24677dcc914e95eb180./info:regioninfo/1731912499883/Put/seqid=0 2024-11-18T06:48:19,989 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/a1458fd8cbb3435b8d5780329ab9ae5c 2024-11-18T06:48:19,997 INFO [regionserver/75ed3c3b0308:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:19,998 INFO [regionserver/75ed3c3b0308:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:20,000 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6858467b2b944fdcbb8afb184c811a59 2024-11-18T06:48:20,015 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/ns/6858467b2b944fdcbb8afb184c811a59 is 43, key is default/ns:d/1731912497265/Put/seqid=0 2024-11-18T06:48:20,016 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/ns/6858467b2b944fdcbb8afb184c811a59 2024-11-18T06:48:20,027 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 58533613db5d49cdb4e5f7ffad6bf927 2024-11-18T06:48:20,044 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/table/58533613db5d49cdb4e5f7ffad6bf927 is 78, key is testBlockEvictionOnRegionMove1731912497536/table:state/1731912497976/Put/seqid=0 2024-11-18T06:48:20,045 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/table/58533613db5d49cdb4e5f7ffad6bf927 2024-11-18T06:48:20,049 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/a1458fd8cbb3435b8d5780329ab9ae5c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/a1458fd8cbb3435b8d5780329ab9ae5c 2024-11-18T06:48:20,051 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/a1458fd8cbb3435b8d5780329ab9ae5c, delay=947 ms 2024-11-18T06:48:20,052 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/a1458fd8cbb3435b8d5780329ab9ae5c, entries=15, sequenceid=15, filesize=7.6 K 2024-11-18T06:48:20,053 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/ns/6858467b2b944fdcbb8afb184c811a59 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/6858467b2b944fdcbb8afb184c811a59 2024-11-18T06:48:20,055 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/6858467b2b944fdcbb8afb184c811a59, delay=949 ms 2024-11-18T06:48:20,056 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/6858467b2b944fdcbb8afb184c811a59, entries=2, sequenceid=15, filesize=5.0 K 2024-11-18T06:48:20,057 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/table/58533613db5d49cdb4e5f7ffad6bf927 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/58533613db5d49cdb4e5f7ffad6bf927 2024-11-18T06:48:20,058 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/58533613db5d49cdb4e5f7ffad6bf927, delay=938 ms 2024-11-18T06:48:20,059 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/58533613db5d49cdb4e5f7ffad6bf927, entries=2, sequenceid=15, filesize=5.3 K 2024-11-18T06:48:20,060 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=15, compaction requested=false 2024-11-18T06:48:20,060 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T06:48:20,061 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/a1458fd8cbb3435b8d5780329ab9ae5c 2024-11-18T06:48:20,062 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/6858467b2b944fdcbb8afb184c811a59 2024-11-18T06:48:20,062 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/58533613db5d49cdb4e5f7ffad6bf927 2024-11-18T06:48:20,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:20,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45391-0x10129d556f00002, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:20,067 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-18T06:48:20,067 INFO [RS:1;75ed3c3b0308:45391 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T06:48:20,067 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T06:48:20,067 INFO [RS:1;75ed3c3b0308:45391 {}] regionserver.HRegionServer(1031): Exiting; stopping=75ed3c3b0308,45391,1731912495403; zookeeper connection closed. 2024-11-18T06:48:20,068 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4040d175 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4040d175 2024-11-18T06:48:20,068 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:20,068 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731912499954Running coprocessor pre-close hooks at 1731912499954Disabling compacts and flushes for region at 1731912499954Disabling writes for close at 1731912499954Obtaining lock to block concurrent updates at 1731912499955 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731912499955Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731912499955Flushing stores of hbase:meta,,1.1588230740 at 1731912499956 (+1 ms)Flushing 1588230740/info: creating writer at 1731912499956Flushing 1588230740/info: appending metadata at 1731912499986 (+30 ms)Flushing 1588230740/info: closing flushed file at 1731912499986Flushing 1588230740/ns: creating writer at 1731912499992 (+6 ms)Flushing 1588230740/ns: appending metadata at 1731912500013 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731912500013Flushing 1588230740/table: creating writer at 1731912500019 (+6 ms)Flushing 1588230740/table: appending metadata at 1731912500043 (+24 ms)Flushing 1588230740/table: closing flushed file at 1731912500043Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14d2b696: reopening flushed file at 1731912500048 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6edac547: reopening flushed file at 1731912500053 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59620f93: reopening flushed file at 1731912500056 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=15, compaction requested=false at 1731912500060 (+4 ms)Writing region close event to WAL at 1731912500063 (+3 ms)Running coprocessor post-close hooks at 1731912500067 (+4 ms)Closed at 1731912500068 (+1 ms) 2024-11-18T06:48:20,068 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:20,154 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(976): stopping server 75ed3c3b0308,43355,1731912495245; all regions closed. 2024-11-18T06:48:20,157 DEBUG [RS:0;75ed3c3b0308:43355 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:20,157 INFO [RS:0;75ed3c3b0308:43355 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C43355%2C1731912495245.meta:.meta(num 1731912497140) 2024-11-18T06:48:20,159 DEBUG [RS:0;75ed3c3b0308:43355 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:20,159 INFO [RS:0;75ed3c3b0308:43355 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C43355%2C1731912495245:(num 1731912496757) 2024-11-18T06:48:20,159 DEBUG [RS:0;75ed3c3b0308:43355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:20,159 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:20,159 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T06:48:20,159 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.ChoreService(370): Chore service for: regionserver/75ed3c3b0308:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T06:48:20,160 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T06:48:20,160 INFO [regionserver/75ed3c3b0308:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T06:48:20,160 INFO [RS:0;75ed3c3b0308:43355 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43355 2024-11-18T06:48:20,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/75ed3c3b0308,43355,1731912495245 2024-11-18T06:48:20,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T06:48:20,163 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T06:48:20,166 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [75ed3c3b0308,43355,1731912495245] 2024-11-18T06:48:20,168 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/75ed3c3b0308,43355,1731912495245 already deleted, retry=false 2024-11-18T06:48:20,168 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 75ed3c3b0308,43355,1731912495245 expired; onlineServers=0 2024-11-18T06:48:20,168 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '75ed3c3b0308,41365,1731912494484' ***** 2024-11-18T06:48:20,168 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T06:48:20,168 INFO [M:0;75ed3c3b0308:41365 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T06:48:20,168 INFO [M:0;75ed3c3b0308:41365 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T06:48:20,169 DEBUG [M:0;75ed3c3b0308:41365 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T06:48:20,169 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T06:48:20,169 DEBUG [M:0;75ed3c3b0308:41365 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T06:48:20,169 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.small.0-1731912496415 {}] cleaner.HFileCleaner(306): Exit Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.small.0-1731912496415,5,FailOnTimeoutGroup] 2024-11-18T06:48:20,169 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.large.0-1731912496410 {}] cleaner.HFileCleaner(306): Exit Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.large.0-1731912496410,5,FailOnTimeoutGroup] 2024-11-18T06:48:20,169 INFO [M:0;75ed3c3b0308:41365 {}] hbase.ChoreService(370): Chore service for: master/75ed3c3b0308:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T06:48:20,169 INFO [M:0;75ed3c3b0308:41365 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T06:48:20,169 DEBUG [M:0;75ed3c3b0308:41365 {}] master.HMaster(1795): Stopping service threads 2024-11-18T06:48:20,170 INFO [M:0;75ed3c3b0308:41365 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T06:48:20,170 INFO [M:0;75ed3c3b0308:41365 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T06:48:20,170 INFO [M:0;75ed3c3b0308:41365 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T06:48:20,171 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T06:48:20,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T06:48:20,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,171 DEBUG [M:0;75ed3c3b0308:41365 {}] zookeeper.ZKUtil(347): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T06:48:20,171 WARN [M:0;75ed3c3b0308:41365 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T06:48:20,172 INFO [M:0;75ed3c3b0308:41365 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.lastflushedseqids 2024-11-18T06:48:20,184 INFO [M:0;75ed3c3b0308:41365 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T06:48:20,184 INFO [M:0;75ed3c3b0308:41365 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T06:48:20,184 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T06:48:20,184 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,184 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,184 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T06:48:20,184 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,185 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-18T06:48:20,195 DEBUG [M:0;75ed3c3b0308:41365 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 025596f41b6b4b348f6767a7d0609092 2024-11-18T06:48:20,208 DEBUG [M:0;75ed3c3b0308:41365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/025596f41b6b4b348f6767a7d0609092 is 82, key is hbase:meta,,1/info:regioninfo/1731912497203/Put/seqid=0 2024-11-18T06:48:20,209 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/025596f41b6b4b348f6767a7d0609092 2024-11-18T06:48:20,218 DEBUG [M:0;75ed3c3b0308:41365 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ea984eb31d7a4a4e849da837af7b99fe 2024-11-18T06:48:20,231 DEBUG [M:0;75ed3c3b0308:41365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea984eb31d7a4a4e849da837af7b99fe is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731912497982/Put/seqid=0 2024-11-18T06:48:20,232 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea984eb31d7a4a4e849da837af7b99fe 2024-11-18T06:48:20,240 DEBUG [M:0;75ed3c3b0308:41365 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 214df18374684a569a5dd9a49db8c058 2024-11-18T06:48:20,253 DEBUG [M:0;75ed3c3b0308:41365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/214df18374684a569a5dd9a49db8c058 is 69, key is 75ed3c3b0308,43355,1731912495245/rs:state/1731912496517/Put/seqid=0 2024-11-18T06:48:20,254 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/214df18374684a569a5dd9a49db8c058 2024-11-18T06:48:20,256 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/025596f41b6b4b348f6767a7d0609092 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/025596f41b6b4b348f6767a7d0609092 2024-11-18T06:48:20,257 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/025596f41b6b4b348f6767a7d0609092, entries=8, sequenceid=84, filesize=5.5 K 2024-11-18T06:48:20,258 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea984eb31d7a4a4e849da837af7b99fe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ea984eb31d7a4a4e849da837af7b99fe 2024-11-18T06:48:20,259 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ea984eb31d7a4a4e849da837af7b99fe, entries=9, sequenceid=84, filesize=6.5 K 2024-11-18T06:48:20,259 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/214df18374684a569a5dd9a49db8c058 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/214df18374684a569a5dd9a49db8c058 2024-11-18T06:48:20,261 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/214df18374684a569a5dd9a49db8c058, entries=2, sequenceid=84, filesize=5.1 K 2024-11-18T06:48:20,262 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false 2024-11-18T06:48:20,264 INFO [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,264 DEBUG [M:0;75ed3c3b0308:41365 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731912500184Disabling compacts and flushes for region at 1731912500184Disabling writes for close at 1731912500184Obtaining lock to block concurrent updates at 1731912500185 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731912500185Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1731912500185Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731912500185Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731912500185Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731912500208 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731912500208Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731912500211 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731912500231 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731912500231Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731912500234 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731912500253 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731912500253Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1731912500255 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1731912500257 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1731912500259 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false at 1731912500262 (+3 ms)Writing region close event to WAL at 1731912500264 (+2 ms)Closed at 1731912500264 2024-11-18T06:48:20,266 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T06:48:20,266 INFO [M:0;75ed3c3b0308:41365 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T06:48:20,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:20,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43355-0x10129d556f00001, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:20,266 INFO [RS:0;75ed3c3b0308:43355 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T06:48:20,266 INFO [M:0;75ed3c3b0308:41365 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41365 2024-11-18T06:48:20,266 INFO [RS:0;75ed3c3b0308:43355 {}] regionserver.HRegionServer(1031): Exiting; stopping=75ed3c3b0308,43355,1731912495245; zookeeper connection closed. 2024-11-18T06:48:20,266 INFO [M:0;75ed3c3b0308:41365 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T06:48:20,267 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47d30d44 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47d30d44 2024-11-18T06:48:20,268 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-18T06:48:20,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:20,370 INFO [M:0;75ed3c3b0308:41365 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T06:48:20,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41365-0x10129d556f00000, quorum=127.0.0.1:52139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:20,374 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T06:48:20,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T06:48:20,395 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/75ed3c3b0308:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/75ed3c3b0308:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731912500051 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731912498349 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731912500055 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/75ed3c3b0308:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731912499870 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/75ed3c3b0308:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1260 (was 1427), ProcessCount=11 (was 11), AvailableMemoryMB=5455 (was 5801) 2024-11-18T06:48:20,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1260, ProcessCount=11, AvailableMemoryMB=5455 2024-11-18T06:48:20,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/hadoop-log-dir so I do NOT create it in target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df 2024-11-18T06:48:20,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/hadoop-tmp-dir so I do NOT create it in target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df 2024-11-18T06:48:20,412 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,416 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/cluster_1d2daea8-deff-6ac2-4cc7-49fd733df315, deleteOnExit=true 2024-11-18T06:48:20,421 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/cluster_1d2daea8-deff-6ac2-4cc7-49fd733df315/zookeeper_0, clientPort=57520, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/cluster_1d2daea8-deff-6ac2-4cc7-49fd733df315/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/cluster_1d2daea8-deff-6ac2-4cc7-49fd733df315/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T06:48:20,422 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57520 2024-11-18T06:48:20,437 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e with version=8 2024-11-18T06:48:20,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/hbase-staging 2024-11-18T06:48:20,439 INFO [Time-limited test {}] client.ConnectionUtils(128): master/75ed3c3b0308:0 server-side Connection retries=45 2024-11-18T06:48:20,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,440 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T06:48:20,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T06:48:20,440 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T06:48:20,440 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T06:48:20,441 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36985 2024-11-18T06:48:20,442 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36985 connecting to ZooKeeper ensemble=127.0.0.1:57520 2024-11-18T06:48:20,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369850x0, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T06:48:20,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36985-0x10129d56f520000 connected 2024-11-18T06:48:20,471 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,471 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:20,473 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e, hbase.cluster.distributed=false 2024-11-18T06:48:20,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T06:48:20,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36985 2024-11-18T06:48:20,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36985 2024-11-18T06:48:20,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36985 2024-11-18T06:48:20,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36985 2024-11-18T06:48:20,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36985 2024-11-18T06:48:20,497 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/75ed3c3b0308:0 server-side Connection retries=45 2024-11-18T06:48:20,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,497 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T06:48:20,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T06:48:20,497 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T06:48:20,497 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T06:48:20,498 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35043 2024-11-18T06:48:20,499 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35043 connecting to ZooKeeper ensemble=127.0.0.1:57520 2024-11-18T06:48:20,500 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,500 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350430x0, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T06:48:20,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:350430x0, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:20,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35043-0x10129d56f520001 connected 2024-11-18T06:48:20,503 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T06:48:20,504 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.cache 2024-11-18T06:48:20,504 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T06:48:20,504 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T06:48:20,505 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T06:48:20,505 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T06:48:20,505 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T06:48:20,505 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T06:48:20,506 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence is complete. 2024-11-18T06:48:20,506 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T06:48:20,508 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T06:48:20,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T06:48:20,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T06:48:20,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35043 2024-11-18T06:48:20,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35043 2024-11-18T06:48:20,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35043 2024-11-18T06:48:20,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35043 2024-11-18T06:48:20,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35043 2024-11-18T06:48:20,535 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/75ed3c3b0308:0 server-side Connection retries=45 2024-11-18T06:48:20,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,536 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T06:48:20,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:20,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T06:48:20,536 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T06:48:20,536 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T06:48:20,537 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46483 2024-11-18T06:48:20,539 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46483 connecting to ZooKeeper ensemble=127.0.0.1:57520 2024-11-18T06:48:20,539 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,539 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464830x0, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T06:48:20,543 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46483-0x10129d56f520002 connected 2024-11-18T06:48:20,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:20,543 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T06:48:20,544 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.cache 2024-11-18T06:48:20,544 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T06:48:20,545 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T06:48:20,545 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T06:48:20,553 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T06:48:20,553 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T06:48:20,553 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T06:48:20,554 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T06:48:20,554 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence is complete. 2024-11-18T06:48:20,554 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T06:48:20,555 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T06:48:20,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T06:48:20,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46483 2024-11-18T06:48:20,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46483 2024-11-18T06:48:20,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46483 2024-11-18T06:48:20,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46483 2024-11-18T06:48:20,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46483 2024-11-18T06:48:20,579 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;75ed3c3b0308:36985 2024-11-18T06:48:20,580 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:20,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:20,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:20,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:20,583 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:20,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T06:48:20,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,587 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T06:48:20,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T06:48:20,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,588 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/75ed3c3b0308,36985,1731912500439 from backup master directory 2024-11-18T06:48:20,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:20,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:20,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:20,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T06:48:20,589 WARN [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T06:48:20,589 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:20,591 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/hbase.id] with ID: 1ea1e81e-d17c-46d3-b6d4-b025bcfcfe7f 2024-11-18T06:48:20,591 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.tmp/hbase.id 2024-11-18T06:48:20,595 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/hbase.id] 2024-11-18T06:48:20,642 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:20,642 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T06:48:20,644 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-18T06:48:20,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,653 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T06:48:20,654 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T06:48:20,654 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:20,677 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store 2024-11-18T06:48:20,683 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:20,684 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T06:48:20,684 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,684 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,684 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T06:48:20,684 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,684 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:20,684 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731912500683Disabling compacts and flushes for region at 1731912500683Disabling writes for close at 1731912500684 (+1 ms)Writing region close event to WAL at 1731912500684Closed at 1731912500684 2024-11-18T06:48:20,684 WARN [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/.initializing 2024-11-18T06:48:20,684 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/WALs/75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:20,690 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T06:48:20,694 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=75ed3c3b0308%2C36985%2C1731912500439, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/WALs/75ed3c3b0308,36985,1731912500439, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/oldWALs, maxLogs=10 2024-11-18T06:48:20,711 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/WALs/75ed3c3b0308,36985,1731912500439/75ed3c3b0308%2C36985%2C1731912500439.1731912500694 2024-11-18T06:48:20,712 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:20,712 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:20,712 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:20,713 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,713 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T06:48:20,718 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:20,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T06:48:20,722 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:20,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T06:48:20,726 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:20,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T06:48:20,730 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:20,730 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,731 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,731 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,731 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,731 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,731 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T06:48:20,732 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T06:48:20,739 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T06:48:20,739 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72961219, jitterRate=0.0872068852186203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T06:48:20,741 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731912500713Initializing all the Stores at 1731912500713Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912500713Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912500713Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912500713Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912500713Cleaning up temporary data from old regions at 1731912500731 (+18 ms)Region opened successfully at 1731912500740 (+9 ms) 2024-11-18T06:48:20,741 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T06:48:20,745 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@789f0889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=75ed3c3b0308/172.17.0.2:0 2024-11-18T06:48:20,747 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T06:48:20,747 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T06:48:20,747 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T06:48:20,747 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T06:48:20,748 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T06:48:20,749 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-18T06:48:20,749 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T06:48:20,751 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T06:48:20,752 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T06:48:20,753 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T06:48:20,754 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T06:48:20,755 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T06:48:20,757 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T06:48:20,757 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T06:48:20,758 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T06:48:20,759 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T06:48:20,761 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T06:48:20,762 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T06:48:20,764 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T06:48:20,766 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T06:48:20,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:20,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:20,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:20,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,769 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=75ed3c3b0308,36985,1731912500439, sessionid=0x10129d56f520000, setting cluster-up flag (Was=false) 2024-11-18T06:48:20,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,779 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T06:48:20,780 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:20,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:20,789 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T06:48:20,790 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:20,791 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T06:48:20,793 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:20,793 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T06:48:20,793 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T06:48:20,794 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 75ed3c3b0308,36985,1731912500439 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=5, maxPoolSize=5 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/75ed3c3b0308:0, corePoolSize=10, maxPoolSize=10 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=2, maxPoolSize=2 2024-11-18T06:48:20,797 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,802 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:20,802 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T06:48:20,802 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,803 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731912530803 2024-11-18T06:48:20,803 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T06:48:20,803 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T06:48:20,803 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T06:48:20,803 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T06:48:20,803 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T06:48:20,803 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T06:48:20,803 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T06:48:20,804 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,804 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T06:48:20,805 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T06:48:20,805 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T06:48:20,805 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T06:48:20,805 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T06:48:20,808 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.large.0-1731912500805,5,FailOnTimeoutGroup] 2024-11-18T06:48:20,811 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.small.0-1731912500808,5,FailOnTimeoutGroup] 2024-11-18T06:48:20,811 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,812 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T06:48:20,812 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,812 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,823 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T06:48:20,824 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:20,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:20,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T06:48:20,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T06:48:20,837 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:20,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T06:48:20,841 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T06:48:20,841 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:20,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T06:48:20,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T06:48:20,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:20,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T06:48:20,849 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T06:48:20,849 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:20,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:20,849 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T06:48:20,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:20,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:20,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T06:48:20,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T06:48:20,851 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T06:48:20,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T06:48:20,858 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T06:48:20,859 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62391745, jitterRate=-0.07029055058956146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T06:48:20,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731912500832Initializing all the Stores at 1731912500832Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912500832Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912500833 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912500833Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912500833Cleaning up temporary data from old regions at 1731912500850 (+17 ms)Region opened successfully at 1731912500859 (+9 ms) 2024-11-18T06:48:20,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T06:48:20,860 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T06:48:20,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T06:48:20,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T06:48:20,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T06:48:20,860 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:20,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731912500860Disabling compacts and flushes for region at 1731912500860Disabling writes for close at 1731912500860Writing region close event to WAL at 1731912500860Closed at 1731912500860 2024-11-18T06:48:20,861 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:20,861 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T06:48:20,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T06:48:20,863 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T06:48:20,864 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T06:48:20,870 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(746): ClusterId : 1ea1e81e-d17c-46d3-b6d4-b025bcfcfe7f 2024-11-18T06:48:20,870 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(746): ClusterId : 1ea1e81e-d17c-46d3-b6d4-b025bcfcfe7f 2024-11-18T06:48:20,870 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T06:48:20,870 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T06:48:20,873 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T06:48:20,873 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T06:48:20,873 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T06:48:20,873 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T06:48:20,877 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T06:48:20,877 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T06:48:20,878 DEBUG [RS:1;75ed3c3b0308:46483 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23035661, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=75ed3c3b0308/172.17.0.2:0 2024-11-18T06:48:20,878 DEBUG [RS:0;75ed3c3b0308:35043 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64eef473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=75ed3c3b0308/172.17.0.2:0 2024-11-18T06:48:20,894 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;75ed3c3b0308:46483 2024-11-18T06:48:20,894 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;75ed3c3b0308:35043 2024-11-18T06:48:20,894 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T06:48:20,894 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T06:48:20,895 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T06:48:20,895 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T06:48:20,895 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T06:48:20,895 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T06:48:20,895 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(2659): reportForDuty to master=75ed3c3b0308,36985,1731912500439 with port=46483, startcode=1731912500535 2024-11-18T06:48:20,895 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(2659): reportForDuty to master=75ed3c3b0308,36985,1731912500439 with port=35043, startcode=1731912500496 2024-11-18T06:48:20,896 DEBUG [RS:1;75ed3c3b0308:46483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T06:48:20,896 DEBUG [RS:0;75ed3c3b0308:35043 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T06:48:20,898 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52447, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T06:48:20,898 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56447, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T06:48:20,899 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36985 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:20,899 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36985 {}] master.ServerManager(517): Registering regionserver=75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:20,900 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36985 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:20,900 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36985 {}] master.ServerManager(517): Registering regionserver=75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:20,900 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:20,900 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T06:48:20,901 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T06:48:20,901 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:20,901 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T06:48:20,902 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T06:48:20,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T06:48:20,910 DEBUG [RS:1;75ed3c3b0308:46483 {}] zookeeper.ZKUtil(111): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:20,910 WARN [RS:1;75ed3c3b0308:46483 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T06:48:20,910 INFO [RS:1;75ed3c3b0308:46483 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:20,910 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [75ed3c3b0308,35043,1731912500496] 2024-11-18T06:48:20,910 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:20,910 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [75ed3c3b0308,46483,1731912500535] 2024-11-18T06:48:20,910 DEBUG [RS:0;75ed3c3b0308:35043 {}] zookeeper.ZKUtil(111): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:20,910 WARN [RS:0;75ed3c3b0308:35043 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T06:48:20,911 INFO [RS:0;75ed3c3b0308:35043 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:20,911 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:20,916 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T06:48:20,916 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T06:48:20,919 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T06:48:20,920 INFO [RS:1;75ed3c3b0308:46483 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T06:48:20,921 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,921 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T06:48:20,922 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T06:48:20,922 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/75ed3c3b0308:0, corePoolSize=2, maxPoolSize=2 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,922 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,923 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,923 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,923 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,923 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,923 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:20,923 DEBUG [RS:1;75ed3c3b0308:46483 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:20,924 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T06:48:20,929 INFO [RS:0;75ed3c3b0308:35043 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T06:48:20,929 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,929 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,929 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,929 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,929 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,929 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T06:48:20,929 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,929 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,46483,1731912500535-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T06:48:20,930 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T06:48:20,930 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,930 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,930 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,930 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,930 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/75ed3c3b0308:0, corePoolSize=2, maxPoolSize=2 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:20,931 DEBUG [RS:0;75ed3c3b0308:35043 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:20,932 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,932 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,932 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,932 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,932 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,932 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,35043,1731912500496-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T06:48:20,945 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T06:48:20,946 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,46483,1731912500535-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,946 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,946 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.Replication(171): 75ed3c3b0308,46483,1731912500535 started 2024-11-18T06:48:20,947 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T06:48:20,947 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,35043,1731912500496-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,947 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,947 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.Replication(171): 75ed3c3b0308,35043,1731912500496 started 2024-11-18T06:48:20,960 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,960 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1482): Serving as 75ed3c3b0308,46483,1731912500535, RpcServer on 75ed3c3b0308/172.17.0.2:46483, sessionid=0x10129d56f520002 2024-11-18T06:48:20,960 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T06:48:20,960 DEBUG [RS:1;75ed3c3b0308:46483 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:20,960 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,46483,1731912500535' 2024-11-18T06:48:20,960 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T06:48:20,961 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T06:48:20,961 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:20,962 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1482): Serving as 75ed3c3b0308,35043,1731912500496, RpcServer on 75ed3c3b0308/172.17.0.2:35043, sessionid=0x10129d56f520001 2024-11-18T06:48:20,962 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T06:48:20,962 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T06:48:20,962 DEBUG [RS:0;75ed3c3b0308:35043 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:20,962 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T06:48:20,962 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,35043,1731912500496' 2024-11-18T06:48:20,962 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T06:48:20,962 DEBUG [RS:1;75ed3c3b0308:46483 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:20,962 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,46483,1731912500535' 2024-11-18T06:48:20,962 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T06:48:20,963 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T06:48:20,963 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T06:48:20,963 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T06:48:20,963 DEBUG [RS:1;75ed3c3b0308:46483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T06:48:20,963 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T06:48:20,963 DEBUG [RS:0;75ed3c3b0308:35043 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:20,963 INFO [RS:1;75ed3c3b0308:46483 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T06:48:20,963 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,35043,1731912500496' 2024-11-18T06:48:20,963 INFO [RS:1;75ed3c3b0308:46483 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T06:48:20,963 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T06:48:20,964 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T06:48:20,965 DEBUG [RS:0;75ed3c3b0308:35043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T06:48:20,965 INFO [RS:0;75ed3c3b0308:35043 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T06:48:20,965 INFO [RS:0;75ed3c3b0308:35043 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T06:48:21,014 WARN [75ed3c3b0308:36985 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T06:48:21,064 INFO [RS:1;75ed3c3b0308:46483 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T06:48:21,065 INFO [RS:0;75ed3c3b0308:35043 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T06:48:21,065 INFO [RS:1;75ed3c3b0308:46483 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C46483%2C1731912500535, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,46483,1731912500535, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:21,066 INFO [RS:0;75ed3c3b0308:35043 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C35043%2C1731912500496, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:21,086 INFO [RS:0;75ed3c3b0308:35043 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496/75ed3c3b0308%2C35043%2C1731912500496.1731912501068 2024-11-18T06:48:21,087 DEBUG [RS:0;75ed3c3b0308:35043 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:21,092 INFO [RS:1;75ed3c3b0308:46483 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,46483,1731912500535/75ed3c3b0308%2C46483%2C1731912500535.1731912501068 2024-11-18T06:48:21,093 DEBUG [RS:1;75ed3c3b0308:46483 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:21,264 DEBUG [75ed3c3b0308:36985 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T06:48:21,265 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(204): Hosts are {75ed3c3b0308=0} racks are {/default-rack=0} 2024-11-18T06:48:21,266 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T06:48:21,266 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T06:48:21,266 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T06:48:21,266 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T06:48:21,266 INFO [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T06:48:21,266 INFO [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T06:48:21,266 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T06:48:21,267 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:21,268 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 75ed3c3b0308,35043,1731912500496, state=OPENING 2024-11-18T06:48:21,270 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T06:48:21,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:21,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:21,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:21,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:21,272 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T06:48:21,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=75ed3c3b0308,35043,1731912500496}] 2024-11-18T06:48:21,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:21,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:21,426 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T06:48:21,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35727, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T06:48:21,430 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T06:48:21,430 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:21,431 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T06:48:21,432 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C35043%2C1731912500496.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:21,455 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496/75ed3c3b0308%2C35043%2C1731912500496.meta.1731912501433.meta 2024-11-18T06:48:21,456 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:21,456 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:21,456 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T06:48:21,457 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T06:48:21,457 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T06:48:21,457 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T06:48:21,457 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:21,457 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T06:48:21,457 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T06:48:21,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T06:48:21,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T06:48:21,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:21,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:21,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T06:48:21,460 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T06:48:21,460 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:21,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:21,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T06:48:21,461 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T06:48:21,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:21,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:21,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T06:48:21,462 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T06:48:21,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:21,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:21,462 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T06:48:21,462 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:21,463 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:21,463 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T06:48:21,463 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T06:48:21,463 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T06:48:21,464 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T06:48:21,464 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60634795, jitterRate=-0.0964711457490921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T06:48:21,464 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T06:48:21,465 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731912501457Writing region info on filesystem at 1731912501457Initializing all the Stores at 1731912501458 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912501458Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912501458Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912501458Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912501458Cleaning up temporary data from old regions at 1731912501463 (+5 ms)Running coprocessor post-open hooks at 1731912501464 (+1 ms)Region opened successfully at 1731912501465 (+1 ms) 2024-11-18T06:48:21,466 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731912501425 2024-11-18T06:48:21,469 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T06:48:21,469 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:21,469 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T06:48:21,469 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 75ed3c3b0308,35043,1731912500496, state=OPEN 2024-11-18T06:48:21,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:21,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:21,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:21,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:21,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:21,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:21,472 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:21,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T06:48:21,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=75ed3c3b0308,35043,1731912500496 in 199 msec 2024-11-18T06:48:21,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T06:48:21,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec 2024-11-18T06:48:21,476 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T06:48:21,476 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T06:48:21,480 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T06:48:21,480 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,35043,1731912500496, seqNum=-1] 2024-11-18T06:48:21,480 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T06:48:21,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55521, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T06:48:21,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 695 msec 2024-11-18T06:48:21,488 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731912501488, completionTime=-1 2024-11-18T06:48:21,488 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T06:48:21,489 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T06:48:21,491 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T06:48:21,491 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731912561491 2024-11-18T06:48:21,491 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731912621491 2024-11-18T06:48:21,491 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-18T06:48:21,492 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,36985,1731912500439-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:21,492 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,36985,1731912500439-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:21,492 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,36985,1731912500439-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:21,492 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-75ed3c3b0308:36985, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:21,492 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:21,492 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:21,495 DEBUG [master/75ed3c3b0308:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.908sec 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,36985,1731912500439-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T06:48:21,498 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,36985,1731912500439-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T06:48:21,499 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T06:48:21,499 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T06:48:21,499 INFO [master/75ed3c3b0308:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,36985,1731912500439-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:21,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T06:48:21,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 75ed3c3b0308,36985,-1 for getting cluster id 2024-11-18T06:48:21,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T06:48:21,572 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1ea1e81e-d17c-46d3-b6d4-b025bcfcfe7f' 2024-11-18T06:48:21,573 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T06:48:21,573 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1ea1e81e-d17c-46d3-b6d4-b025bcfcfe7f" 2024-11-18T06:48:21,573 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64008405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T06:48:21,573 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [75ed3c3b0308,36985,-1] 2024-11-18T06:48:21,574 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T06:48:21,574 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:21,576 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T06:48:21,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T06:48:21,577 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T06:48:21,579 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,35043,1731912500496, seqNum=-1] 2024-11-18T06:48:21,579 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T06:48:21,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T06:48:21,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:21,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T06:48:21,585 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:21,586 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b176c3 2024-11-18T06:48:21,586 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T06:48:21,588 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T06:48:21,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36985 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731912501584', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T06:48:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36985 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 2024-11-18T06:48:21,591 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T06:48:21,591 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:21,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36985 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731912501584" procId is: 4 2024-11-18T06:48:21,592 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T06:48:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36985 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:21,604 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fbadb4a25dc0394e0224e75f34b7ff54, NAME => 'testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731912501584', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:21,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:21,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(1722): Closing fbadb4a25dc0394e0224e75f34b7ff54, disabling compactions & flushes 2024-11-18T06:48:21,612 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:21,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:21,613 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. after waiting 0 ms 2024-11-18T06:48:21,613 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:21,613 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:21,613 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731912501584-pool-0 {}] regionserver.HRegion(1676): Region close journal for fbadb4a25dc0394e0224e75f34b7ff54: Waiting for close lock at 1731912501612Disabling compacts and flushes for region at 1731912501612Disabling writes for close at 1731912501613 (+1 ms)Writing region close event to WAL at 1731912501613Closed at 1731912501613 2024-11-18T06:48:21,614 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T06:48:21,614 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731912501614"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731912501614"}]},"ts":"1731912501614"} 2024-11-18T06:48:21,616 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T06:48:21,617 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T06:48:21,617 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731912501584","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731912501617"}]},"ts":"1731912501617"} 2024-11-18T06:48:21,620 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731912501584, state=ENABLING in hbase:meta 2024-11-18T06:48:21,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {75ed3c3b0308=0} racks are {/default-rack=0} 2024-11-18T06:48:21,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T06:48:21,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T06:48:21,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T06:48:21,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T06:48:21,620 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T06:48:21,620 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T06:48:21,620 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T06:48:21,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN}] 2024-11-18T06:48:21,622 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN 2024-11-18T06:48:21,623 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN; state=OFFLINE, location=75ed3c3b0308,35043,1731912500496; forceNewPlan=false, retain=false 2024-11-18T06:48:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36985 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:21,773 INFO [75ed3c3b0308:36985 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T06:48:21,774 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fbadb4a25dc0394e0224e75f34b7ff54, regionState=OPENING, regionLocation=75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:21,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN because future has completed 2024-11-18T06:48:21,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbadb4a25dc0394e0224e75f34b7ff54, server=75ed3c3b0308,35043,1731912500496}] 2024-11-18T06:48:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36985 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:21,932 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:21,933 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fbadb4a25dc0394e0224e75f34b7ff54, NAME => 'testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:21,933 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731912501584 fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,933 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:21,933 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,933 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,934 INFO [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,938 INFO [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbadb4a25dc0394e0224e75f34b7ff54 columnFamilyName family 2024-11-18T06:48:21,938 DEBUG [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:21,938 INFO [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] regionserver.HStore(327): Store=fbadb4a25dc0394e0224e75f34b7ff54/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:21,939 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,939 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,939 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,940 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,940 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,940 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,946 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T06:48:21,947 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fbadb4a25dc0394e0224e75f34b7ff54; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67883741, jitterRate=0.01154656708240509}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T06:48:21,947 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:21,948 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fbadb4a25dc0394e0224e75f34b7ff54: Running coprocessor pre-open hook at 1731912501933Writing region info on filesystem at 1731912501933Initializing all the Stores at 1731912501934 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912501934Cleaning up temporary data from old regions at 1731912501940 (+6 ms)Running coprocessor post-open hooks at 1731912501947 (+7 ms)Region opened successfully at 1731912501948 (+1 ms) 2024-11-18T06:48:21,949 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54., pid=6, masterSystemTime=1731912501930 2024-11-18T06:48:21,951 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:21,951 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:21,952 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fbadb4a25dc0394e0224e75f34b7ff54, regionState=OPEN, openSeqNum=2, regionLocation=75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:21,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbadb4a25dc0394e0224e75f34b7ff54, server=75ed3c3b0308,35043,1731912500496 because future has completed 2024-11-18T06:48:21,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T06:48:21,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fbadb4a25dc0394e0224e75f34b7ff54, server=75ed3c3b0308,35043,1731912500496 in 179 msec 2024-11-18T06:48:21,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T06:48:21,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN in 336 msec 2024-11-18T06:48:21,960 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T06:48:21,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731912501584","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731912501960"}]},"ts":"1731912501960"} 2024-11-18T06:48:21,962 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731912501584, state=ENABLED in hbase:meta 2024-11-18T06:48:21,963 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T06:48:21,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731912501584 in 374 msec 2024-11-18T06:48:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36985 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T06:48:22,218 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731912501584 completed 2024-11-18T06:48:22,218 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731912501584 get assigned. Timeout = 60000ms 2024-11-18T06:48:22,218 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T06:48:22,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731912501584 assigned to meta. Checking AM states. 2024-11-18T06:48:22,222 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T06:48:22,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731912501584 assigned. 2024-11-18T06:48:22,225 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731912501584', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54., hostname=75ed3c3b0308,35043,1731912500496, seqNum=2] 2024-11-18T06:48:22,229 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fbadb4a25dc0394e0224e75f34b7ff54 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T06:48:22,239 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 33b36638673741cbb4ad814cf1fa844a 2024-11-18T06:48:22,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/.tmp/family/33b36638673741cbb4ad814cf1fa844a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T06:48:22,253 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/.tmp/family/33b36638673741cbb4ad814cf1fa844a 2024-11-18T06:48:22,256 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/.tmp/family/33b36638673741cbb4ad814cf1fa844a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a 2024-11-18T06:48:22,257 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a, delay=955 ms 2024-11-18T06:48:22,258 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T06:48:22,258 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for fbadb4a25dc0394e0224e75f34b7ff54 in 29ms, sequenceid=6, compaction requested=false 2024-11-18T06:48:22,259 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731912501584' 2024-11-18T06:48:22,259 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fbadb4a25dc0394e0224e75f34b7ff54: 2024-11-18T06:48:22,675 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T06:48:23,213 DEBUG [hfile-prefetch-1731912499870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 33b36638673741cbb4ad814cf1fa844a_0. 2024-11-18T06:48:23,214 DEBUG [hfile-prefetch-1731912499870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 33b36638673741cbb4ad814cf1fa844a_117. 2024-11-18T06:48:23,214 DEBUG [hfile-prefetch-1731912499870 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a, with total blocks 2, and data blocks 1 2024-11-18T06:48:23,214 DEBUG [hfile-prefetch-1731912499870 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T06:48:23,214 DEBUG [hfile-prefetch-1731912499870 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T06:48:23,214 DEBUG [hfile-prefetch-1731912499870 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a. Will sleep 100ms and try the verification again. 2024-11-18T06:48:23,250 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T06:48:23,250 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T06:48:23,250 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:23,251 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:23,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T06:48:23,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731912503250 2024-11-18T06:48:23,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T06:48:23,259 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;75ed3c3b0308:35043,5,FailOnTimeoutGroup] 2024-11-18T06:48:23,260 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '75ed3c3b0308,35043,1731912500496' ***** 2024-11-18T06:48:23,260 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-18T06:48:23,260 INFO [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T06:48:23,260 INFO [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence 2024-11-18T06:48:23,260 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T06:48:23,260 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T06:48:23,260 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T06:48:23,260 INFO [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T06:48:23,260 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T06:48:23,270 DEBUG [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1487): Thread RS:0;75ed3c3b0308:35043 started persisting bucket cache to file 2024-11-18T06:48:23,271 DEBUG [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:23,271 DEBUG [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:23,278 DEBUG [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T06:48:23,278 DEBUG [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731912503270 2024-11-18T06:48:23,278 DEBUG [RS:0;75ed3c3b0308:35043 {}] bucket.BucketCache(1509): Thread RS:0;75ed3c3b0308:35043 finished persisting bucket cache to file, renaming 2024-11-18T06:48:23,278 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T06:48:23,278 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T06:48:23,278 INFO [RS:0;75ed3c3b0308:35043 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(3091): Received CLOSE for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(959): stopping server 75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;75ed3c3b0308:35043. 2024-11-18T06:48:23,279 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T06:48:23,279 DEBUG [RS:0;75ed3c3b0308:35043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:23,279 DEBUG [RS:0;75ed3c3b0308:35043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:23,279 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fbadb4a25dc0394e0224e75f34b7ff54, disabling compactions & flushes 2024-11-18T06:48:23,279 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:23,279 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:23,279 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. after waiting 0 ms 2024-11-18T06:48:23,279 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T06:48:23,279 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T06:48:23,280 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T06:48:23,280 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T06:48:23,280 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, fbadb4a25dc0394e0224e75f34b7ff54=testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.} 2024-11-18T06:48:23,280 DEBUG [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:23,280 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T06:48:23,281 DEBUG [StoreFileCloser-fbadb4a25dc0394e0224e75f34b7ff54-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a 2024-11-18T06:48:23,281 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T06:48:23,281 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T06:48:23,281 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T06:48:23,281 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T06:48:23,281 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T06:48:23,281 WARN [hfile-prefetch-1731912499870 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T06:48:23,281 DEBUG [hfile-prefetch-1731912499870 {}] hfile.PrefetchExecutor(121): Prefetch completed for 33b36638673741cbb4ad814cf1fa844a 2024-11-18T06:48:23,281 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-18T06:48:23,285 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T06:48:23,286 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:23,286 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fbadb4a25dc0394e0224e75f34b7ff54: Waiting for close lock at 1731912503279Running coprocessor pre-close hooks at 1731912503279Disabling compacts and flushes for region at 1731912503279Disabling writes for close at 1731912503279Writing region close event to WAL at 1731912503281 (+2 ms)Running coprocessor post-close hooks at 1731912503286 (+5 ms)Closed at 1731912503286 2024-11-18T06:48:23,286 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:23,295 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ad90c6774c4d42379e44213aaaa7e533 2024-11-18T06:48:23,312 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/ad90c6774c4d42379e44213aaaa7e533 is 209, key is testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54./info:regioninfo/1731912501952/Put/seqid=0 2024-11-18T06:48:23,312 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/ad90c6774c4d42379e44213aaaa7e533 2024-11-18T06:48:23,321 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f07a4118cfb444c3920315395b4ae0f3 2024-11-18T06:48:23,334 INFO [regionserver/75ed3c3b0308:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:23,335 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/ns/f07a4118cfb444c3920315395b4ae0f3 is 43, key is default/ns:d/1731912501483/Put/seqid=0 2024-11-18T06:48:23,336 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/ns/f07a4118cfb444c3920315395b4ae0f3 2024-11-18T06:48:23,346 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 09c36c87e6a44de0b034bdeac6791164 2024-11-18T06:48:23,359 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/table/09c36c87e6a44de0b034bdeac6791164 is 80, key is testBlockEvictionOnGracefulStop1731912501584/table:state/1731912501960/Put/seqid=0 2024-11-18T06:48:23,360 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/table/09c36c87e6a44de0b034bdeac6791164 2024-11-18T06:48:23,364 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/ad90c6774c4d42379e44213aaaa7e533 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/ad90c6774c4d42379e44213aaaa7e533 2024-11-18T06:48:23,366 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/ad90c6774c4d42379e44213aaaa7e533, delay=925 ms 2024-11-18T06:48:23,367 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/ad90c6774c4d42379e44213aaaa7e533, entries=10, sequenceid=11, filesize=7.0 K 2024-11-18T06:48:23,368 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/ns/f07a4118cfb444c3920315395b4ae0f3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/f07a4118cfb444c3920315395b4ae0f3 2024-11-18T06:48:23,369 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/f07a4118cfb444c3920315395b4ae0f3, delay=900 ms 2024-11-18T06:48:23,371 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/f07a4118cfb444c3920315395b4ae0f3, entries=2, sequenceid=11, filesize=5.0 K 2024-11-18T06:48:23,371 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/table/09c36c87e6a44de0b034bdeac6791164 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/09c36c87e6a44de0b034bdeac6791164 2024-11-18T06:48:23,373 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/09c36c87e6a44de0b034bdeac6791164, delay=909 ms 2024-11-18T06:48:23,374 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/09c36c87e6a44de0b034bdeac6791164, entries=2, sequenceid=11, filesize=5.3 K 2024-11-18T06:48:23,375 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=11, compaction requested=false 2024-11-18T06:48:23,375 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/ad90c6774c4d42379e44213aaaa7e533 2024-11-18T06:48:23,376 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/f07a4118cfb444c3920315395b4ae0f3 2024-11-18T06:48:23,377 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/09c36c87e6a44de0b034bdeac6791164 2024-11-18T06:48:23,381 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-18T06:48:23,381 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T06:48:23,381 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:23,381 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731912503280Running coprocessor pre-close hooks at 1731912503280Disabling compacts and flushes for region at 1731912503281 (+1 ms)Disabling writes for close at 1731912503281Obtaining lock to block concurrent updates at 1731912503281Preparing flush snapshotting stores in 1588230740 at 1731912503281Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731912503282 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731912503282Flushing 1588230740/info: creating writer at 1731912503282Flushing 1588230740/info: appending metadata at 1731912503311 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731912503311Flushing 1588230740/ns: creating writer at 1731912503315 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731912503335 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731912503335Flushing 1588230740/table: creating writer at 1731912503338 (+3 ms)Flushing 1588230740/table: appending metadata at 1731912503358 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731912503359 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dc350e8: reopening flushed file at 1731912503363 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a488dc8: reopening flushed file at 1731912503367 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ec5c8ad: reopening flushed file at 1731912503371 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=11, compaction requested=false at 1731912503375 (+4 ms)Writing region close event to WAL at 1731912503377 (+2 ms)Running coprocessor post-close hooks at 1731912503381 (+4 ms)Closed at 1731912503381 2024-11-18T06:48:23,381 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:23,481 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(976): stopping server 75ed3c3b0308,35043,1731912500496; all regions closed. 2024-11-18T06:48:23,483 DEBUG [RS:0;75ed3c3b0308:35043 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:23,483 INFO [RS:0;75ed3c3b0308:35043 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C35043%2C1731912500496.meta:.meta(num 1731912501433) 2024-11-18T06:48:23,485 DEBUG [RS:0;75ed3c3b0308:35043 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:23,485 INFO [RS:0;75ed3c3b0308:35043 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C35043%2C1731912500496:(num 1731912501068) 2024-11-18T06:48:23,485 DEBUG [RS:0;75ed3c3b0308:35043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:23,485 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:23,485 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T06:48:23,485 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.ChoreService(370): Chore service for: regionserver/75ed3c3b0308:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T06:48:23,485 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T06:48:23,485 INFO [regionserver/75ed3c3b0308:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T06:48:23,486 INFO [RS:0;75ed3c3b0308:35043 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35043 2024-11-18T06:48:23,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/75ed3c3b0308,35043,1731912500496 2024-11-18T06:48:23,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T06:48:23,490 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T06:48:23,492 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [75ed3c3b0308,35043,1731912500496] 2024-11-18T06:48:23,494 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/75ed3c3b0308,35043,1731912500496 already deleted, retry=false 2024-11-18T06:48:23,494 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 75ed3c3b0308,35043,1731912500496 on 75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:23,498 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 75ed3c3b0308,35043,1731912500496, splitWal=true, meta=true 2024-11-18T06:48:23,501 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 75ed3c3b0308,35043,1731912500496 (carryingMeta=true) 75ed3c3b0308,35043,1731912500496/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@8d395f7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T06:48:23,501 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 75ed3c3b0308,35043,1731912500496, splitWal=true, meta=true 2024-11-18T06:48:23,502 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 75ed3c3b0308,35043,1731912500496, splitWal=true, meta=true, isMeta: true 2024-11-18T06:48:23,502 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496-splitting 2024-11-18T06:48:23,503 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496-splitting dir is empty, no logs to split. 2024-11-18T06:48:23,503 INFO [PEWorker-2 {}] master.SplitWALManager(105): 75ed3c3b0308,35043,1731912500496 WAL count=0, meta=true 2024-11-18T06:48:23,505 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496-splitting dir is empty, no logs to split. 2024-11-18T06:48:23,505 INFO [PEWorker-2 {}] master.SplitWALManager(105): 75ed3c3b0308,35043,1731912500496 WAL count=0, meta=true 2024-11-18T06:48:23,505 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 75ed3c3b0308,35043,1731912500496 WAL splitting is done? wals=0, meta=true 2024-11-18T06:48:23,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T06:48:23,507 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T06:48:23,508 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T06:48:23,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:23,592 INFO [RS:0;75ed3c3b0308:35043 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T06:48:23,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35043-0x10129d56f520001, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:23,592 INFO [RS:0;75ed3c3b0308:35043 {}] regionserver.HRegionServer(1031): Exiting; stopping=75ed3c3b0308,35043,1731912500496; zookeeper connection closed. 2024-11-18T06:48:23,592 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e7338b9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e7338b9 2024-11-18T06:48:23,658 DEBUG [75ed3c3b0308:36985 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T06:48:23,659 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:23,660 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 75ed3c3b0308,46483,1731912500535, state=OPENING 2024-11-18T06:48:23,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:23,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:23,662 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T06:48:23,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:23,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:23,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=75ed3c3b0308,46483,1731912500535}] 2024-11-18T06:48:23,778 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/75ed3c3b0308:0 server-side Connection retries=45 2024-11-18T06:48:23,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:23,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:23,778 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T06:48:23,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T06:48:23,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T06:48:23,778 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T06:48:23,778 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T06:48:23,779 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35087 2024-11-18T06:48:23,780 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35087 connecting to ZooKeeper ensemble=127.0.0.1:57520 2024-11-18T06:48:23,780 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:23,781 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:23,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350870x0, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T06:48:23,784 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35087-0x10129d56f520003 connected 2024-11-18T06:48:23,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-18T06:48:23,784 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-18T06:48:23,785 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T06:48:23,785 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.cache 2024-11-18T06:48:23,785 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T06:48:23,786 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T06:48:23,786 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T06:48:23,788 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T06:48:23,788 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T06:48:23,789 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T06:48:23,789 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-18T06:48:23,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T06:48:23,791 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-18T06:48:23,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T06:48:23,796 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-18T06:48:23,797 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T06:48:23,797 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-18T06:48:23,797 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence is complete. 2024-11-18T06:48:23,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35087 2024-11-18T06:48:23,801 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35087 2024-11-18T06:48:23,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35087 2024-11-18T06:48:23,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35087 2024-11-18T06:48:23,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35087 2024-11-18T06:48:23,806 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(746): ClusterId : 1ea1e81e-d17c-46d3-b6d4-b025bcfcfe7f 2024-11-18T06:48:23,806 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T06:48:23,809 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T06:48:23,809 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T06:48:23,814 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T06:48:23,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36341, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T06:48:23,817 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T06:48:23,817 DEBUG [RS:2;75ed3c3b0308:35087 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67512097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=75ed3c3b0308/172.17.0.2:0 2024-11-18T06:48:23,818 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T06:48:23,818 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:23,819 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T06:48:23,819 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C46483%2C1731912500535.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,46483,1731912500535, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:23,835 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;75ed3c3b0308:35087 2024-11-18T06:48:23,835 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T06:48:23,835 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T06:48:23,835 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T06:48:23,836 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(2659): reportForDuty to master=75ed3c3b0308,36985,1731912500439 with port=35087, startcode=1731912503777 2024-11-18T06:48:23,837 DEBUG [RS:2;75ed3c3b0308:35087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T06:48:23,839 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,46483,1731912500535/75ed3c3b0308%2C46483%2C1731912500535.meta.1731912503820.meta 2024-11-18T06:48:23,839 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:23,839 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46319, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T06:48:23,840 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:23,840 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T06:48:23,840 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T06:48:23,840 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T06:48:23,840 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36985 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:23,840 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T06:48:23,840 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36985 {}] master.ServerManager(517): Registering regionserver=75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:23,840 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:23,841 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T06:48:23,841 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T06:48:23,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T06:48:23,842 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e 2024-11-18T06:48:23,842 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T06:48:23,842 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T06:48:23,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T06:48:23,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:23,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T06:48:23,845 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/ad90c6774c4d42379e44213aaaa7e533, delay=948 ms 2024-11-18T06:48:23,845 DEBUG [RS:2;75ed3c3b0308:35087 {}] zookeeper.ZKUtil(111): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:23,845 WARN [RS:2;75ed3c3b0308:35087 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T06:48:23,845 INFO [RS:2;75ed3c3b0308:35087 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T06:48:23,845 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [75ed3c3b0308,35087,1731912503777] 2024-11-18T06:48:23,845 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:23,846 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/ad90c6774c4d42379e44213aaaa7e533 2024-11-18T06:48:23,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:23,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T06:48:23,847 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T06:48:23,847 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:23,849 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/f07a4118cfb444c3920315395b4ae0f3, delay=970 ms 2024-11-18T06:48:23,850 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/f07a4118cfb444c3920315395b4ae0f3 2024-11-18T06:48:23,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:23,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T06:48:23,851 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T06:48:23,851 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:23,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:23,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T06:48:23,852 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T06:48:23,853 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:23,853 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T06:48:23,854 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/09c36c87e6a44de0b034bdeac6791164, delay=952 ms 2024-11-18T06:48:23,856 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/09c36c87e6a44de0b034bdeac6791164 2024-11-18T06:48:23,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T06:48:23,856 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T06:48:23,856 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T06:48:23,857 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:23,857 INFO [RS:2;75ed3c3b0308:35087 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T06:48:23,857 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,857 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740 2024-11-18T06:48:23,857 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T06:48:23,858 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T06:48:23,858 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T06:48:23,858 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T06:48:23,858 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,858 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,858 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,858 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,858 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,858 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/75ed3c3b0308:0, corePoolSize=2, maxPoolSize=2 2024-11-18T06:48:23,859 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/75ed3c3b0308:0, corePoolSize=1, maxPoolSize=1 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:23,859 DEBUG [RS:2;75ed3c3b0308:35087 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/75ed3c3b0308:0, corePoolSize=3, maxPoolSize=3 2024-11-18T06:48:23,859 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T06:48:23,860 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,860 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,860 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,860 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,860 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,860 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60435292, jitterRate=-0.09944397211074829}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T06:48:23,860 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,35087,1731912503777-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T06:48:23,860 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T06:48:23,861 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731912503841Writing region info on filesystem at 1731912503841Initializing all the Stores at 1731912503841Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912503841Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912503842 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912503842Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731912503842Cleaning up temporary data from old regions at 1731912503858 (+16 ms)Running coprocessor post-open hooks at 1731912503860 (+2 ms)Region opened successfully at 1731912503861 (+1 ms) 2024-11-18T06:48:23,862 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731912503814 2024-11-18T06:48:23,865 DEBUG [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T06:48:23,865 INFO [RS_OPEN_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T06:48:23,865 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:23,866 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 75ed3c3b0308,46483,1731912500535, state=OPEN 2024-11-18T06:48:23,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:23,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:23,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:23,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:23,868 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:23,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-18T06:48:23,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=75ed3c3b0308,46483,1731912500535 in 207 msec 2024-11-18T06:48:23,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T06:48:23,873 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T06:48:23,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T06:48:23,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 365 msec 2024-11-18T06:48:23,874 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 75ed3c3b0308,35043,1731912500496 had 2 regions 2024-11-18T06:48:23,875 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 75ed3c3b0308,35043,1731912500496, splitWal=true, meta=true, isMeta: false 2024-11-18T06:48:23,876 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496-splitting dir is empty, no logs to split. 2024-11-18T06:48:23,876 INFO [PEWorker-3 {}] master.SplitWALManager(105): 75ed3c3b0308,35043,1731912500496 WAL count=0, meta=false 2024-11-18T06:48:23,877 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35043,1731912500496-splitting dir is empty, no logs to split. 2024-11-18T06:48:23,877 INFO [PEWorker-3 {}] master.SplitWALManager(105): 75ed3c3b0308,35043,1731912500496 WAL count=0, meta=false 2024-11-18T06:48:23,877 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 75ed3c3b0308,35043,1731912500496 WAL splitting is done? wals=0, meta=false 2024-11-18T06:48:23,878 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T06:48:23,879 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=75ed3c3b0308,35087,1731912503777-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,879 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,879 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.Replication(171): 75ed3c3b0308,35087,1731912503777 started 2024-11-18T06:48:23,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN}] 2024-11-18T06:48:23,880 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN 2024-11-18T06:48:23,881 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T06:48:23,900 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T06:48:23,901 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1482): Serving as 75ed3c3b0308,35087,1731912503777, RpcServer on 75ed3c3b0308/172.17.0.2:35087, sessionid=0x10129d56f520003 2024-11-18T06:48:23,901 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T06:48:23,901 DEBUG [RS:2;75ed3c3b0308:35087 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:23,901 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,35087,1731912503777' 2024-11-18T06:48:23,901 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T06:48:23,902 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T06:48:23,902 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T06:48:23,902 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T06:48:23,902 DEBUG [RS:2;75ed3c3b0308:35087 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:23,902 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '75ed3c3b0308,35087,1731912503777' 2024-11-18T06:48:23,902 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T06:48:23,903 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T06:48:23,903 DEBUG [RS:2;75ed3c3b0308:35087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T06:48:23,903 INFO [RS:2;75ed3c3b0308:35087 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T06:48:23,904 INFO [RS:2;75ed3c3b0308:35087 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T06:48:24,004 INFO [RS:2;75ed3c3b0308:35087 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T06:48:24,005 INFO [RS:2;75ed3c3b0308:35087 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=75ed3c3b0308%2C35087%2C1731912503777, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35087,1731912503777, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs, maxLogs=55 2024-11-18T06:48:24,027 INFO [RS:2;75ed3c3b0308:35087 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/WALs/75ed3c3b0308,35087,1731912503777/75ed3c3b0308%2C35087%2C1731912503777.1731912504006 2024-11-18T06:48:24,027 DEBUG [RS:2;75ed3c3b0308:35087 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T06:48:24,032 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(204): Hosts are {75ed3c3b0308=0} racks are {/default-rack=0} 2024-11-18T06:48:24,032 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T06:48:24,032 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T06:48:24,032 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T06:48:24,032 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T06:48:24,032 INFO [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T06:48:24,032 INFO [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T06:48:24,033 DEBUG [75ed3c3b0308:36985 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T06:48:24,033 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fbadb4a25dc0394e0224e75f34b7ff54, regionState=OPENING, regionLocation=75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:24,034 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 75ed3c3b0308:35043 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 75ed3c3b0308/172.17.0.2:35043 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T06:48:24,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,35043,1731912500496, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,35043,1731912500496, seqNum=-1, error=java.net.ConnectException: Call to address=75ed3c3b0308:35043 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 75ed3c3b0308/172.17.0.2:35043 2024-11-18T06:48:24,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,35043,1731912500496, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T06:48:24,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,35043,1731912500496, seqNum=-1 from cache 2024-11-18T06:48:24,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 75ed3c3b0308:35043 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 75ed3c3b0308/172.17.0.2:35043 2024-11-18T06:48:24,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T06:48:24,146 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=75ed3c3b0308,46483,1731912500535, seqNum=-1] 2024-11-18T06:48:24,146 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T06:48:24,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57191, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T06:48:24,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN because future has completed 2024-11-18T06:48:24,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbadb4a25dc0394e0224e75f34b7ff54, server=75ed3c3b0308,35087,1731912503777}] 2024-11-18T06:48:24,303 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T06:48:24,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60573, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T06:48:24,307 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,307 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => fbadb4a25dc0394e0224e75f34b7ff54, NAME => 'testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.', STARTKEY => '', ENDKEY => ''} 2024-11-18T06:48:24,308 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731912501584 fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,308 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T06:48:24,308 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,308 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,309 INFO [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,309 INFO [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbadb4a25dc0394e0224e75f34b7ff54 columnFamilyName family 2024-11-18T06:48:24,310 DEBUG [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T06:48:24,311 DEBUG [StoreFileOpener-fbadb4a25dc0394e0224e75f34b7ff54-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a, delay=912 ms 2024-11-18T06:48:24,312 DEBUG [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a 2024-11-18T06:48:24,312 INFO [StoreOpener-fbadb4a25dc0394e0224e75f34b7ff54-1 {}] regionserver.HStore(327): Store=fbadb4a25dc0394e0224e75f34b7ff54/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T06:48:24,312 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,313 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,314 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,314 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,314 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,315 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,315 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened fbadb4a25dc0394e0224e75f34b7ff54; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66587874, jitterRate=-0.00776335597038269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T06:48:24,316 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,317 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for fbadb4a25dc0394e0224e75f34b7ff54: Running coprocessor pre-open hook at 1731912504308Writing region info on filesystem at 1731912504308Initializing all the Stores at 1731912504308Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731912504308Cleaning up temporary data from old regions at 1731912504314 (+6 ms)Running coprocessor post-open hooks at 1731912504316 (+2 ms)Region opened successfully at 1731912504317 (+1 ms) 2024-11-18T06:48:24,317 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54., pid=11, masterSystemTime=1731912504303 2024-11-18T06:48:24,320 DEBUG [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,320 INFO [RS_OPEN_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fbadb4a25dc0394e0224e75f34b7ff54, regionState=OPEN, openSeqNum=10, regionLocation=75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:24,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbadb4a25dc0394e0224e75f34b7ff54, server=75ed3c3b0308,35087,1731912503777 because future has completed 2024-11-18T06:48:24,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-18T06:48:24,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fbadb4a25dc0394e0224e75f34b7ff54, server=75ed3c3b0308,35087,1731912503777 in 174 msec 2024-11-18T06:48:24,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-18T06:48:24,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731912501584, region=fbadb4a25dc0394e0224e75f34b7ff54, ASSIGN in 446 msec 2024-11-18T06:48:24,327 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 75ed3c3b0308,35043,1731912500496 after splitting done 2024-11-18T06:48:24,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 75ed3c3b0308,35043,1731912500496, splitWal=true, meta=true in 832 msec 2024-11-18T06:48:24,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T06:48:24,401 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T06:48:24,401 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:24,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:24,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:24,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T06:48:24,402 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T06:48:24,402 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1082213907, stopped=false 2024-11-18T06:48:24,402 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=75ed3c3b0308,36985,1731912500439 2024-11-18T06:48:24,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:24,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:24,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T06:48:24,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:24,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:24,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:24,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T06:48:24,404 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T06:48:24,404 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:24,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:24,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:24,405 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:24,405 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '75ed3c3b0308,46483,1731912500535' ***** 2024-11-18T06:48:24,405 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T06:48:24,405 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '75ed3c3b0308,35087,1731912503777' ***** 2024-11-18T06:48:24,405 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T06:48:24,405 INFO [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T06:48:24,405 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T06:48:24,406 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T06:48:24,406 INFO [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence 2024-11-18T06:48:24,406 INFO [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T06:48:24,406 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T06:48:24,406 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T06:48:24,406 INFO [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/751dcb62-cba8-fbd6-18ae-f5563742f7df/bucket.persistence 2024-11-18T06:48:24,406 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T06:48:24,406 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T06:48:24,406 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T06:48:24,406 INFO [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T06:48:24,406 INFO [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T06:48:24,407 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T06:48:24,407 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T06:48:24,417 DEBUG [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1487): Thread RS:1;75ed3c3b0308:46483 started persisting bucket cache to file 2024-11-18T06:48:24,417 DEBUG [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1487): Thread RS:2;75ed3c3b0308:35087 started persisting bucket cache to file 2024-11-18T06:48:24,417 DEBUG [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:24,417 DEBUG [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:24,417 DEBUG [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T06:48:24,417 DEBUG [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T06:48:24,420 DEBUG [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T06:48:24,420 DEBUG [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731912504417 2024-11-18T06:48:24,420 DEBUG [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T06:48:24,420 DEBUG [RS:1;75ed3c3b0308:46483 {}] bucket.BucketCache(1509): Thread RS:1;75ed3c3b0308:46483 finished persisting bucket cache to file, renaming 2024-11-18T06:48:24,420 DEBUG [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731912504417 2024-11-18T06:48:24,420 DEBUG [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1509): Thread RS:2;75ed3c3b0308:35087 finished persisting bucket cache to file, renaming 2024-11-18T06:48:24,421 WARN [RS:2;75ed3c3b0308:35087 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T06:48:24,421 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T06:48:24,421 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T06:48:24,421 INFO [RS:2;75ed3c3b0308:35087 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T06:48:24,421 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(959): stopping server 75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:24,421 INFO [RS:2;75ed3c3b0308:35087 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T06:48:24,421 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(3091): Received CLOSE for fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;75ed3c3b0308:46483. 2024-11-18T06:48:24,421 DEBUG [RS:1;75ed3c3b0308:46483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:24,421 DEBUG [RS:1;75ed3c3b0308:46483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:24,421 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(959): stopping server 75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:24,421 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T06:48:24,421 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T06:48:24,421 INFO [RS:2;75ed3c3b0308:35087 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;75ed3c3b0308:35087. 2024-11-18T06:48:24,422 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T06:48:24,422 DEBUG [RS:2;75ed3c3b0308:35087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T06:48:24,422 DEBUG [RS:2;75ed3c3b0308:35087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:24,422 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fbadb4a25dc0394e0224e75f34b7ff54, disabling compactions & flushes 2024-11-18T06:48:24,422 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T06:48:24,422 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T06:48:24,422 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,422 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1325): Online Regions={fbadb4a25dc0394e0224e75f34b7ff54=testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54.} 2024-11-18T06:48:24,422 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,422 DEBUG [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1351): Waiting on fbadb4a25dc0394e0224e75f34b7ff54 2024-11-18T06:48:24,422 DEBUG [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T06:48:24,422 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. after waiting 0 ms 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,422 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T06:48:24,422 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T06:48:24,422 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-18T06:48:24,423 DEBUG [StoreFileCloser-fbadb4a25dc0394e0224e75f34b7ff54-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/family/33b36638673741cbb4ad814cf1fa844a 2024-11-18T06:48:24,427 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/default/testBlockEvictionOnGracefulStop1731912501584/fbadb4a25dc0394e0224e75f34b7ff54/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T06:48:24,428 INFO [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,428 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fbadb4a25dc0394e0224e75f34b7ff54: Waiting for close lock at 1731912504421Running coprocessor pre-close hooks at 1731912504422 (+1 ms)Disabling compacts and flushes for region at 1731912504422Disabling writes for close at 1731912504422Writing region close event to WAL at 1731912504423 (+1 ms)Running coprocessor post-close hooks at 1731912504428 (+5 ms)Closed at 1731912504428 2024-11-18T06:48:24,428 DEBUG [RS_CLOSE_REGION-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54. 2024-11-18T06:48:24,432 INFO [regionserver/75ed3c3b0308:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:24,433 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e8debba101b94beea8ae082da7b46ce1 2024-11-18T06:48:24,447 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/e8debba101b94beea8ae082da7b46ce1 is 209, key is testBlockEvictionOnGracefulStop1731912501584,,1731912501589.fbadb4a25dc0394e0224e75f34b7ff54./info:regioninfo/1731912504320/Put/seqid=0 2024-11-18T06:48:24,448 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/e8debba101b94beea8ae082da7b46ce1 2024-11-18T06:48:24,450 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/.tmp/info/e8debba101b94beea8ae082da7b46ce1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/e8debba101b94beea8ae082da7b46ce1 2024-11-18T06:48:24,451 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/e8debba101b94beea8ae082da7b46ce1, delay=952 ms 2024-11-18T06:48:24,452 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/e8debba101b94beea8ae082da7b46ce1, entries=8, sequenceid=19, filesize=6.6 K 2024-11-18T06:48:24,453 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-18T06:48:24,454 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/ad90c6774c4d42379e44213aaaa7e533 2024-11-18T06:48:24,454 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/info/e8debba101b94beea8ae082da7b46ce1 2024-11-18T06:48:24,454 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/ns/f07a4118cfb444c3920315395b4ae0f3 2024-11-18T06:48:24,455 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/table/09c36c87e6a44de0b034bdeac6791164 2024-11-18T06:48:24,460 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-18T06:48:24,460 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T06:48:24,460 INFO [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:24,460 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731912504422Running coprocessor pre-close hooks at 1731912504422Disabling compacts and flushes for region at 1731912504422Disabling writes for close at 1731912504422Obtaining lock to block concurrent updates at 1731912504422Preparing flush snapshotting stores in 1588230740 at 1731912504423 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731912504423Flushing stores of hbase:meta,,1.1588230740 at 1731912504423Flushing 1588230740/info: creating writer at 1731912504423Flushing 1588230740/info: appending metadata at 1731912504447 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731912504447Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ab8357: reopening flushed file at 1731912504450 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1731912504453 (+3 ms)Writing region close event to WAL at 1731912504455 (+2 ms)Running coprocessor post-close hooks at 1731912504460 (+5 ms)Closed at 1731912504460 2024-11-18T06:48:24,460 DEBUG [RS_CLOSE_META-regionserver/75ed3c3b0308:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T06:48:24,461 INFO [regionserver/75ed3c3b0308:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:24,622 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(976): stopping server 75ed3c3b0308,35087,1731912503777; all regions closed. 2024-11-18T06:48:24,622 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(976): stopping server 75ed3c3b0308,46483,1731912500535; all regions closed. 2024-11-18T06:48:24,624 DEBUG [RS:2;75ed3c3b0308:35087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:24,624 INFO [RS:2;75ed3c3b0308:35087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C35087%2C1731912503777:(num 1731912504006) 2024-11-18T06:48:24,624 DEBUG [RS:2;75ed3c3b0308:35087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:24,624 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:24,624 DEBUG [RS:1;75ed3c3b0308:46483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:24,625 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T06:48:24,625 INFO [RS:1;75ed3c3b0308:46483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C46483%2C1731912500535.meta:.meta(num 1731912503820) 2024-11-18T06:48:24,625 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.ChoreService(370): Chore service for: regionserver/75ed3c3b0308:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T06:48:24,625 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T06:48:24,625 INFO [regionserver/75ed3c3b0308:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T06:48:24,625 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T06:48:24,625 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T06:48:24,625 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T06:48:24,625 INFO [RS:2;75ed3c3b0308:35087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35087 2024-11-18T06:48:24,627 DEBUG [RS:1;75ed3c3b0308:46483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/oldWALs 2024-11-18T06:48:24,627 INFO [RS:1;75ed3c3b0308:46483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 75ed3c3b0308%2C46483%2C1731912500535:(num 1731912501068) 2024-11-18T06:48:24,627 DEBUG [RS:1;75ed3c3b0308:46483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T06:48:24,627 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T06:48:24,627 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T06:48:24,627 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.ChoreService(370): Chore service for: regionserver/75ed3c3b0308:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T06:48:24,627 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T06:48:24,627 INFO [regionserver/75ed3c3b0308:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T06:48:24,627 INFO [RS:1;75ed3c3b0308:46483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46483 2024-11-18T06:48:24,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T06:48:24,631 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T06:48:24,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/75ed3c3b0308,35087,1731912503777 2024-11-18T06:48:24,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/75ed3c3b0308,46483,1731912500535 2024-11-18T06:48:24,632 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T06:48:24,633 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [75ed3c3b0308,35087,1731912503777] 2024-11-18T06:48:24,635 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/75ed3c3b0308,35087,1731912503777 already deleted, retry=false 2024-11-18T06:48:24,635 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 75ed3c3b0308,35087,1731912503777 expired; onlineServers=1 2024-11-18T06:48:24,635 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [75ed3c3b0308,46483,1731912500535] 2024-11-18T06:48:24,637 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/75ed3c3b0308,46483,1731912500535 already deleted, retry=false 2024-11-18T06:48:24,637 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 75ed3c3b0308,46483,1731912500535 expired; onlineServers=0 2024-11-18T06:48:24,637 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '75ed3c3b0308,36985,1731912500439' ***** 2024-11-18T06:48:24,637 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T06:48:24,637 INFO [M:0;75ed3c3b0308:36985 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T06:48:24,638 INFO [M:0;75ed3c3b0308:36985 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T06:48:24,638 DEBUG [M:0;75ed3c3b0308:36985 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T06:48:24,638 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T06:48:24,638 DEBUG [M:0;75ed3c3b0308:36985 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T06:48:24,638 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.small.0-1731912500808 {}] cleaner.HFileCleaner(306): Exit Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.small.0-1731912500808,5,FailOnTimeoutGroup] 2024-11-18T06:48:24,638 DEBUG [master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.large.0-1731912500805 {}] cleaner.HFileCleaner(306): Exit Thread[master/75ed3c3b0308:0:becomeActiveMaster-HFileCleaner.large.0-1731912500805,5,FailOnTimeoutGroup] 2024-11-18T06:48:24,638 INFO [M:0;75ed3c3b0308:36985 {}] hbase.ChoreService(370): Chore service for: master/75ed3c3b0308:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T06:48:24,638 INFO [M:0;75ed3c3b0308:36985 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T06:48:24,638 DEBUG [M:0;75ed3c3b0308:36985 {}] master.HMaster(1795): Stopping service threads 2024-11-18T06:48:24,638 INFO [M:0;75ed3c3b0308:36985 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T06:48:24,639 INFO [M:0;75ed3c3b0308:36985 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T06:48:24,639 INFO [M:0;75ed3c3b0308:36985 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T06:48:24,639 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T06:48:24,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T06:48:24,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T06:48:24,639 DEBUG [M:0;75ed3c3b0308:36985 {}] zookeeper.ZKUtil(347): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T06:48:24,640 WARN [M:0;75ed3c3b0308:36985 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T06:48:24,640 INFO [M:0;75ed3c3b0308:36985 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/.lastflushedseqids 2024-11-18T06:48:24,644 INFO [M:0;75ed3c3b0308:36985 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T06:48:24,644 INFO [M:0;75ed3c3b0308:36985 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T06:48:24,644 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T06:48:24,644 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:24,644 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:24,644 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T06:48:24,644 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:24,644 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.49 KB heapSize=49.88 KB 2024-11-18T06:48:24,654 DEBUG [M:0;75ed3c3b0308:36985 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a842e1f114a04876a54a4164f9c6a5a2 2024-11-18T06:48:24,668 DEBUG [M:0;75ed3c3b0308:36985 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a842e1f114a04876a54a4164f9c6a5a2 is 82, key is hbase:meta,,1/info:regioninfo/1731912503865/Put/seqid=0 2024-11-18T06:48:24,669 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a842e1f114a04876a54a4164f9c6a5a2 2024-11-18T06:48:24,677 DEBUG [M:0;75ed3c3b0308:36985 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 469fb49a1fa54d66998c5bd60fdb3ab6 2024-11-18T06:48:24,691 DEBUG [M:0;75ed3c3b0308:36985 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/469fb49a1fa54d66998c5bd60fdb3ab6 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731912501964/Put/seqid=0 2024-11-18T06:48:24,692 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.26 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/469fb49a1fa54d66998c5bd60fdb3ab6 2024-11-18T06:48:24,699 DEBUG [M:0;75ed3c3b0308:36985 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on affb8509932b421fb0d6df60f6c8e5af 2024-11-18T06:48:24,713 DEBUG [M:0;75ed3c3b0308:36985 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/affb8509932b421fb0d6df60f6c8e5af is 69, key is 75ed3c3b0308,35087,1731912503777/rs:state/1731912503841/Put/seqid=0 2024-11-18T06:48:24,714 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/affb8509932b421fb0d6df60f6c8e5af 2024-11-18T06:48:24,716 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for affb8509932b421fb0d6df60f6c8e5af 2024-11-18T06:48:24,716 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a842e1f114a04876a54a4164f9c6a5a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a842e1f114a04876a54a4164f9c6a5a2 2024-11-18T06:48:24,718 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a842e1f114a04876a54a4164f9c6a5a2, entries=14, sequenceid=102, filesize=5.9 K 2024-11-18T06:48:24,718 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/469fb49a1fa54d66998c5bd60fdb3ab6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/469fb49a1fa54d66998c5bd60fdb3ab6 2024-11-18T06:48:24,720 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/469fb49a1fa54d66998c5bd60fdb3ab6, entries=11, sequenceid=102, filesize=6.5 K 2024-11-18T06:48:24,720 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/affb8509932b421fb0d6df60f6c8e5af as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/affb8509932b421fb0d6df60f6c8e5af 2024-11-18T06:48:24,722 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for affb8509932b421fb0d6df60f6c8e5af 2024-11-18T06:48:24,722 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5a781b9-de7a-f13b-b2c1-dce5207bfb17/2af9d024-d861-6470-9e4d-76f873f88f7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/affb8509932b421fb0d6df60f6c8e5af, entries=3, sequenceid=102, filesize=5.3 K 2024-11-18T06:48:24,723 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=102, compaction requested=false 2024-11-18T06:48:24,725 INFO [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T06:48:24,725 DEBUG [M:0;75ed3c3b0308:36985 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731912504644Disabling compacts and flushes for region at 1731912504644Disabling writes for close at 1731912504644Obtaining lock to block concurrent updates at 1731912504644Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731912504644Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40436, getHeapSize=51016, getOffHeapSize=0, getCellsCount=127 at 1731912504645 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731912504645Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731912504645Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731912504668 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731912504668Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731912504671 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731912504691 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731912504691Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731912504693 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731912504712 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731912504713 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195e3871: reopening flushed file at 1731912504716 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@acc9c2: reopening flushed file at 1731912504718 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d2934b5: reopening flushed file at 1731912504720 (+2 ms)Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=102, compaction requested=false at 1731912504723 (+3 ms)Writing region close event to WAL at 1731912504725 (+2 ms)Closed at 1731912504725 2024-11-18T06:48:24,726 INFO [M:0;75ed3c3b0308:36985 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T06:48:24,726 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T06:48:24,726 INFO [M:0;75ed3c3b0308:36985 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36985 2024-11-18T06:48:24,726 INFO [M:0;75ed3c3b0308:36985 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T06:48:24,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:24,733 INFO [RS:2;75ed3c3b0308:35087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T06:48:24,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35087-0x10129d56f520003, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:24,733 INFO [RS:2;75ed3c3b0308:35087 {}] regionserver.HRegionServer(1031): Exiting; stopping=75ed3c3b0308,35087,1731912503777; zookeeper connection closed. 2024-11-18T06:48:24,733 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@615971e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@615971e3 2024-11-18T06:48:24,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:24,734 INFO [RS:1;75ed3c3b0308:46483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T06:48:24,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46483-0x10129d56f520002, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:24,734 INFO [RS:1;75ed3c3b0308:46483 {}] regionserver.HRegionServer(1031): Exiting; stopping=75ed3c3b0308,46483,1731912500535; zookeeper connection closed. 2024-11-18T06:48:24,734 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22c226e9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22c226e9 2024-11-18T06:48:24,734 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-18T06:48:24,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:24,828 INFO [M:0;75ed3c3b0308:36985 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T06:48:24,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36985-0x10129d56f520000, quorum=127.0.0.1:57520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T06:48:24,831 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T06:48:24,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T06:48:24,833 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T06:48:24,839 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1159 (was 1260), ProcessCount=11 (was 11), AvailableMemoryMB=5355 (was 5455)