2024-11-25 04:54:43,353 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 04:54:43,370 main DEBUG Took 0.015577 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 04:54:43,371 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 04:54:43,371 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 04:54:43,372 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 04:54:43,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,383 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 04:54:43,402 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,404 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,405 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,405 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,406 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,408 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,409 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,411 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,411 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,412 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,413 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,414 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,416 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,417 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,418 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 04:54:43,419 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,420 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 04:54:43,422 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 04:54:43,424 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 04:54:43,427 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 04:54:43,428 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 04:54:43,430 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 04:54:43,430 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 04:54:43,443 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 04:54:43,446 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 04:54:43,448 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 04:54:43,449 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 04:54:43,449 main DEBUG createAppenders(={Console}) 2024-11-25 04:54:43,451 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-25 04:54:43,451 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 04:54:43,452 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-25 04:54:43,453 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 04:54:43,453 main DEBUG OutputStream closed 2024-11-25 04:54:43,454 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 04:54:43,454 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 04:54:43,454 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-25 04:54:43,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 04:54:43,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 04:54:43,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 04:54:43,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 04:54:43,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 04:54:43,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 04:54:43,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 04:54:43,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 04:54:43,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 04:54:43,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 04:54:43,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 04:54:43,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 04:54:43,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 04:54:43,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 04:54:43,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 04:54:43,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 04:54:43,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 04:54:43,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 04:54:43,535 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 04:54:43,536 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-25 04:54:43,536 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 04:54:43,537 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-25T04:54:43,552 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-25 04:54:43,555 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 04:54:43,555 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T04:54:43,821 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f 2024-11-25T04:54:43,845 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=479, ProcessCount=11, AvailableMemoryMB=12211 2024-11-25T04:54:43,956 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T04:54:44,076 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:44,093 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/cluster_fae29c37-c96a-3498-c320-7d8fcac115a9, deleteOnExit=true 2024-11-25T04:54:44,238 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/cluster_fae29c37-c96a-3498-c320-7d8fcac115a9/zookeeper_0, clientPort=49566, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/cluster_fae29c37-c96a-3498-c320-7d8fcac115a9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/cluster_fae29c37-c96a-3498-c320-7d8fcac115a9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T04:54:44,250 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49566 2024-11-25T04:54:44,251 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:44,326 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f with version=8 2024-11-25T04:54:44,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/hbase-staging 2024-11-25T04:54:44,442 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T04:54:44,677 INFO [Time-limited test {}] client.ConnectionUtils(128): master/13aa81add8b8:0 server-side Connection retries=45 2024-11-25T04:54:44,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:44,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:44,695 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T04:54:44,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:44,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T04:54:44,855 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T04:54:44,957 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T04:54:45,003 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T04:54:45,016 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T04:54:45,021 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T04:54:45,064 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 95198 (auto-detected) 2024-11-25T04:54:45,066 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T04:54:45,095 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43575 2024-11-25T04:54:45,121 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43575 connecting to ZooKeeper ensemble=127.0.0.1:49566 2024-11-25T04:54:45,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435750x0, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T04:54:45,235 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43575-0x10171a199bd0000 connected 2024-11-25T04:54:45,341 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:45,343 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:45,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:45,358 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f, hbase.cluster.distributed=false 2024-11-25T04:54:45,389 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T04:54:45,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43575 2024-11-25T04:54:45,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43575 2024-11-25T04:54:45,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43575 2024-11-25T04:54:45,405 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43575 2024-11-25T04:54:45,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43575 2024-11-25T04:54:45,499 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/13aa81add8b8:0 server-side Connection retries=45 2024-11-25T04:54:45,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:45,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:45,502 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T04:54:45,502 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:45,502 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T04:54:45,506 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T04:54:45,509 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T04:54:45,510 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32787 2024-11-25T04:54:45,513 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32787 connecting to ZooKeeper ensemble=127.0.0.1:49566 2024-11-25T04:54:45,513 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:45,514 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:45,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:327870x0, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T04:54:45,542 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32787-0x10171a199bd0001 connected 2024-11-25T04:54:45,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:45,549 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T04:54:45,564 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.cache 2024-11-25T04:54:45,572 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T04:54:45,574 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T04:54:45,576 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T04:54:45,578 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T04:54:45,578 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T04:54:45,585 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T04:54:45,588 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T04:54:45,589 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.persistence is complete. 2024-11-25T04:54:45,594 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T04:54:45,597 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T04:54:45,604 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T04:54:45,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32787 2024-11-25T04:54:45,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32787 2024-11-25T04:54:45,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32787 2024-11-25T04:54:45,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32787 2024-11-25T04:54:45,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32787 2024-11-25T04:54:45,654 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/13aa81add8b8:0 server-side Connection retries=45 2024-11-25T04:54:45,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:45,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:45,655 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T04:54:45,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:45,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T04:54:45,656 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T04:54:45,656 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T04:54:45,658 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38993 2024-11-25T04:54:45,661 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38993 connecting to ZooKeeper ensemble=127.0.0.1:49566 2024-11-25T04:54:45,661 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:45,663 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:45,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389930x0, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T04:54:45,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38993-0x10171a199bd0002 connected 2024-11-25T04:54:45,688 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:45,688 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T04:54:45,691 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.cache 2024-11-25T04:54:45,691 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T04:54:45,692 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T04:54:45,692 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T04:54:45,693 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T04:54:45,693 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T04:54:45,693 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T04:54:45,694 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T04:54:45,694 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.persistence is complete. 2024-11-25T04:54:45,694 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T04:54:45,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T04:54:45,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T04:54:45,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38993 2024-11-25T04:54:45,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38993 2024-11-25T04:54:45,725 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38993 2024-11-25T04:54:45,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38993 2024-11-25T04:54:45,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38993 2024-11-25T04:54:45,756 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;13aa81add8b8:43575 2024-11-25T04:54:45,759 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/13aa81add8b8,43575,1732510484520 2024-11-25T04:54:45,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:45,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:45,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:45,786 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/13aa81add8b8,43575,1732510484520 2024-11-25T04:54:45,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T04:54:45,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T04:54:45,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:45,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:45,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:45,827 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T04:54:45,832 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/13aa81add8b8,43575,1732510484520 from backup master directory 2024-11-25T04:54:45,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:45,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:45,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/13aa81add8b8,43575,1732510484520 2024-11-25T04:54:45,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:45,848 WARN [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T04:54:45,849 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=13aa81add8b8,43575,1732510484520 2024-11-25T04:54:45,852 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T04:54:45,854 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T04:54:45,878 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/hbase.id] with ID: f1be64eb-6480-4100-976f-2d3d9f204c71 2024-11-25T04:54:45,879 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.tmp/hbase.id 2024-11-25T04:54:45,888 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/hbase.id] 2024-11-25T04:54:45,949 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:45,957 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T04:54:45,982 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-25T04:54:46,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:46,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:46,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:46,048 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T04:54:46,051 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T04:54:46,106 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T04:54:46,114 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:46,225 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store 2024-11-25T04:54:46,277 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T04:54:46,285 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:46,288 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T04:54:46,288 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:46,288 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:46,296 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T04:54:46,298 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:46,298 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:46,299 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732510486287Disabling compacts and flushes for region at 1732510486287Disabling writes for close at 1732510486297 (+10 ms)Writing region close event to WAL at 1732510486298 (+1 ms)Closed at 1732510486298 2024-11-25T04:54:46,300 WARN [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/.initializing 2024-11-25T04:54:46,300 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/WALs/13aa81add8b8,43575,1732510484520 2024-11-25T04:54:46,334 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T04:54:46,352 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=13aa81add8b8%2C43575%2C1732510484520, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/WALs/13aa81add8b8,43575,1732510484520, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/oldWALs, maxLogs=10 2024-11-25T04:54:46,404 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/WALs/13aa81add8b8,43575,1732510484520/13aa81add8b8%2C43575%2C1732510484520.1732510486359 2024-11-25T04:54:46,407 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:46,408 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:46,409 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:46,414 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,416 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T04:54:46,526 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:46,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:46,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T04:54:46,541 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:46,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:46,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T04:54:46,560 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:46,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:46,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T04:54:46,585 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:46,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:46,586 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,590 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,591 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,595 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,596 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,600 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T04:54:46,604 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:46,613 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T04:54:46,614 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64634315, jitterRate=-0.036873653531074524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T04:54:46,626 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732510486432Initializing all the Stores at 1732510486433 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510486434 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510486435 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510486436 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510486436Cleaning up temporary data from old regions at 1732510486596 (+160 ms)Region opened successfully at 1732510486625 (+29 ms) 2024-11-25T04:54:46,628 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T04:54:46,671 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c82f2fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=13aa81add8b8/172.17.0.2:0 2024-11-25T04:54:46,719 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T04:54:46,739 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T04:54:46,739 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T04:54:46,745 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T04:54:46,746 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T04:54:46,765 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 18 msec 2024-11-25T04:54:46,766 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T04:54:46,825 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T04:54:46,839 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T04:54:46,886 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T04:54:46,889 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T04:54:46,893 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T04:54:46,902 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T04:54:46,906 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T04:54:46,913 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T04:54:46,924 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T04:54:46,927 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T04:54:46,935 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T04:54:46,961 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T04:54:46,969 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T04:54:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,000 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=13aa81add8b8,43575,1732510484520, sessionid=0x10171a199bd0000, setting cluster-up flag (Was=false) 2024-11-25T04:54:47,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,052 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T04:54:47,055 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=13aa81add8b8,43575,1732510484520 2024-11-25T04:54:47,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,102 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T04:54:47,104 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=13aa81add8b8,43575,1732510484520 2024-11-25T04:54:47,109 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T04:54:47,144 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(746): ClusterId : f1be64eb-6480-4100-976f-2d3d9f204c71 2024-11-25T04:54:47,148 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(746): ClusterId : f1be64eb-6480-4100-976f-2d3d9f204c71 2024-11-25T04:54:47,148 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T04:54:47,149 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T04:54:47,188 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T04:54:47,188 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T04:54:47,188 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T04:54:47,188 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T04:54:47,200 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T04:54:47,200 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T04:54:47,200 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:47,201 DEBUG [RS:0;13aa81add8b8:32787 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa056a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=13aa81add8b8/172.17.0.2:0 2024-11-25T04:54:47,201 DEBUG [RS:1;13aa81add8b8:38993 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b3d0a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=13aa81add8b8/172.17.0.2:0 2024-11-25T04:54:47,213 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T04:54:47,218 DEBUG [RS:0;13aa81add8b8:32787 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;13aa81add8b8:32787 2024-11-25T04:54:47,218 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;13aa81add8b8:38993 2024-11-25T04:54:47,220 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T04:54:47,222 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T04:54:47,222 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T04:54:47,222 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T04:54:47,222 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T04:54:47,223 DEBUG [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T04:54:47,223 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T04:54:47,226 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(2659): reportForDuty to master=13aa81add8b8,43575,1732510484520 with port=32787, startcode=1732510485458 2024-11-25T04:54:47,226 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(2659): reportForDuty to master=13aa81add8b8,43575,1732510484520 with port=38993, startcode=1732510485653 2024-11-25T04:54:47,226 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 13aa81add8b8,43575,1732510484520 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T04:54:47,234 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:47,234 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:47,234 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:47,235 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:47,235 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/13aa81add8b8:0, corePoolSize=10, maxPoolSize=10 2024-11-25T04:54:47,235 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,236 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/13aa81add8b8:0, corePoolSize=2, maxPoolSize=2 2024-11-25T04:54:47,236 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,241 DEBUG [RS:1;13aa81add8b8:38993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T04:54:47,241 DEBUG [RS:0;13aa81add8b8:32787 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T04:54:47,243 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:47,244 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T04:54:47,249 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:47,250 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T04:54:47,253 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732510517253 2024-11-25T04:54:47,256 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T04:54:47,258 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T04:54:47,262 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T04:54:47,262 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T04:54:47,263 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T04:54:47,263 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T04:54:47,264 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,275 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T04:54:47,277 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T04:54:47,277 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T04:54:47,283 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T04:54:47,283 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T04:54:47,286 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.large.0-1732510487285,5,FailOnTimeoutGroup] 2024-11-25T04:54:47,288 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.small.0-1732510487286,5,FailOnTimeoutGroup] 2024-11-25T04:54:47,288 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,288 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T04:54:47,289 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,290 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,290 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T04:54:47,290 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:47,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:47,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T04:54:47,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T04:54:47,323 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:47,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:47,323 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48791, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T04:54:47,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T04:54:47,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45471, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T04:54:47,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43575 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 13aa81add8b8,32787,1732510485458 2024-11-25T04:54:47,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T04:54:47,334 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:47,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:47,334 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43575 {}] master.ServerManager(517): Registering regionserver=13aa81add8b8,32787,1732510485458 2024-11-25T04:54:47,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T04:54:47,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T04:54:47,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:47,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:47,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T04:54:47,347 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43575 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 13aa81add8b8,38993,1732510485653 2024-11-25T04:54:47,347 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43575 {}] master.ServerManager(517): Registering regionserver=13aa81add8b8,38993,1732510485653 2024-11-25T04:54:47,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T04:54:47,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:47,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:47,352 DEBUG [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:47,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T04:54:47,353 DEBUG [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T04:54:47,353 DEBUG [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T04:54:47,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:47,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:47,354 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:47,354 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T04:54:47,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T04:54:47,354 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T04:54:47,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T04:54:47,355 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T04:54:47,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T04:54:47,371 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T04:54:47,372 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75431020, jitterRate=0.12400978803634644}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T04:54:47,376 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732510487316Initializing all the Stores at 1732510487316Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510487316Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510487316Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510487316Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510487316Cleaning up temporary data from old regions at 1732510487355 (+39 ms)Region opened successfully at 1732510487376 (+21 ms) 2024-11-25T04:54:47,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T04:54:47,377 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T04:54:47,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T04:54:47,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T04:54:47,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T04:54:47,379 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T04:54:47,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732510487377Disabling compacts and flushes for region at 1732510487377Disabling writes for close at 1732510487377Writing region close event to WAL at 1732510487379 (+2 ms)Closed at 1732510487379 2024-11-25T04:54:47,381 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:47,382 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T04:54:47,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T04:54:47,400 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T04:54:47,402 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T04:54:47,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:47,423 DEBUG [RS:1;13aa81add8b8:38993 {}] zookeeper.ZKUtil(111): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/13aa81add8b8,38993,1732510485653 2024-11-25T04:54:47,424 WARN [RS:1;13aa81add8b8:38993 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T04:54:47,424 INFO [RS:1;13aa81add8b8:38993 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:47,424 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,38993,1732510485653 2024-11-25T04:54:47,424 DEBUG [RS:0;13aa81add8b8:32787 {}] zookeeper.ZKUtil(111): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/13aa81add8b8,32787,1732510485458 2024-11-25T04:54:47,425 WARN [RS:0;13aa81add8b8:32787 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T04:54:47,425 INFO [RS:0;13aa81add8b8:32787 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:47,425 DEBUG [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,32787,1732510485458 2024-11-25T04:54:47,427 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [13aa81add8b8,32787,1732510485458] 2024-11-25T04:54:47,427 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [13aa81add8b8,38993,1732510485653] 2024-11-25T04:54:47,471 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T04:54:47,487 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T04:54:47,496 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T04:54:47,503 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T04:54:47,503 INFO [RS:1;13aa81add8b8:38993 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T04:54:47,503 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,504 INFO [RS:0;13aa81add8b8:32787 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T04:54:47,504 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,512 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T04:54:47,512 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T04:54:47,518 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T04:54:47,518 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T04:54:47,520 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,521 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,521 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,521 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,521 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,521 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,521 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/13aa81add8b8:0, corePoolSize=2, maxPoolSize=2 2024-11-25T04:54:47,522 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/13aa81add8b8:0, corePoolSize=2, maxPoolSize=2 2024-11-25T04:54:47,522 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,522 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:47,523 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,523 DEBUG [RS:0;13aa81add8b8:32787 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:47,523 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:47,524 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:47,524 DEBUG [RS:1;13aa81add8b8:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:47,533 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,534 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,534 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,534 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,534 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,534 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,32787,1732510485458-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T04:54:47,536 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,536 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,537 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,537 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,537 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,537 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,38993,1732510485653-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T04:54:47,554 WARN [13aa81add8b8:43575 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T04:54:47,566 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T04:54:47,566 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T04:54:47,568 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,38993,1732510485653-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,568 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,32787,1732510485458-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,568 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,568 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,569 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.Replication(171): 13aa81add8b8,32787,1732510485458 started 2024-11-25T04:54:47,569 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.Replication(171): 13aa81add8b8,38993,1732510485653 started 2024-11-25T04:54:47,591 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,591 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:47,592 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1482): Serving as 13aa81add8b8,38993,1732510485653, RpcServer on 13aa81add8b8/172.17.0.2:38993, sessionid=0x10171a199bd0002 2024-11-25T04:54:47,592 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(1482): Serving as 13aa81add8b8,32787,1732510485458, RpcServer on 13aa81add8b8/172.17.0.2:32787, sessionid=0x10171a199bd0001 2024-11-25T04:54:47,593 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T04:54:47,593 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T04:54:47,593 DEBUG [RS:1;13aa81add8b8:38993 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 13aa81add8b8,38993,1732510485653 2024-11-25T04:54:47,593 DEBUG [RS:0;13aa81add8b8:32787 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 13aa81add8b8,32787,1732510485458 2024-11-25T04:54:47,594 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,32787,1732510485458' 2024-11-25T04:54:47,594 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,38993,1732510485653' 2024-11-25T04:54:47,594 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T04:54:47,594 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T04:54:47,595 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T04:54:47,595 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T04:54:47,596 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T04:54:47,596 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T04:54:47,596 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T04:54:47,596 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T04:54:47,596 DEBUG [RS:1;13aa81add8b8:38993 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 13aa81add8b8,38993,1732510485653 2024-11-25T04:54:47,596 DEBUG [RS:0;13aa81add8b8:32787 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 13aa81add8b8,32787,1732510485458 2024-11-25T04:54:47,596 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,32787,1732510485458' 2024-11-25T04:54:47,596 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,38993,1732510485653' 2024-11-25T04:54:47,596 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T04:54:47,596 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T04:54:47,597 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T04:54:47,597 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T04:54:47,598 DEBUG [RS:1;13aa81add8b8:38993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T04:54:47,598 DEBUG [RS:0;13aa81add8b8:32787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T04:54:47,598 INFO [RS:0;13aa81add8b8:32787 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T04:54:47,598 INFO [RS:1;13aa81add8b8:38993 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T04:54:47,598 INFO [RS:0;13aa81add8b8:32787 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T04:54:47,598 INFO [RS:1;13aa81add8b8:38993 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T04:54:47,704 INFO [RS:1;13aa81add8b8:38993 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T04:54:47,704 INFO [RS:0;13aa81add8b8:32787 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T04:54:47,706 INFO [RS:0;13aa81add8b8:32787 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=13aa81add8b8%2C32787%2C1732510485458, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,32787,1732510485458, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs, maxLogs=55 2024-11-25T04:54:47,706 INFO [RS:1;13aa81add8b8:38993 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=13aa81add8b8%2C38993%2C1732510485653, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,38993,1732510485653, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs, maxLogs=55 2024-11-25T04:54:47,740 INFO [RS:1;13aa81add8b8:38993 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,38993,1732510485653/13aa81add8b8%2C38993%2C1732510485653.1732510487711 2024-11-25T04:54:47,741 DEBUG [RS:1;13aa81add8b8:38993 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:47,743 INFO [RS:0;13aa81add8b8:32787 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,32787,1732510485458/13aa81add8b8%2C32787%2C1732510485458.1732510487711 2024-11-25T04:54:47,747 DEBUG [RS:0;13aa81add8b8:32787 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:47,807 DEBUG [13aa81add8b8:43575 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T04:54:47,821 DEBUG [13aa81add8b8:43575 {}] balancer.BalancerClusterState(204): Hosts are {13aa81add8b8=0} racks are {/default-rack=0} 2024-11-25T04:54:47,825 DEBUG [13aa81add8b8:43575 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T04:54:47,825 DEBUG [13aa81add8b8:43575 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T04:54:47,825 DEBUG [13aa81add8b8:43575 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T04:54:47,825 DEBUG [13aa81add8b8:43575 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T04:54:47,826 INFO [13aa81add8b8:43575 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T04:54:47,826 INFO [13aa81add8b8:43575 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T04:54:47,826 DEBUG [13aa81add8b8:43575 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T04:54:47,836 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=13aa81add8b8,38993,1732510485653 2024-11-25T04:54:47,843 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 13aa81add8b8,38993,1732510485653, state=OPENING 2024-11-25T04:54:47,910 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T04:54:47,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:47,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:47,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:47,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:47,921 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T04:54:47,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=13aa81add8b8,38993,1732510485653}] 2024-11-25T04:54:48,095 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T04:54:48,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T04:54:48,107 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T04:54:48,107 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:48,108 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T04:54:48,110 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=13aa81add8b8%2C38993%2C1732510485653.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,38993,1732510485653, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs, maxLogs=55 2024-11-25T04:54:48,128 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,38993,1732510485653/13aa81add8b8%2C38993%2C1732510485653.meta.1732510488112.meta 2024-11-25T04:54:48,128 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:48,128 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:48,130 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T04:54:48,132 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T04:54:48,136 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T04:54:48,139 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T04:54:48,140 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:48,140 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T04:54:48,140 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T04:54:48,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T04:54:48,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T04:54:48,143 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:48,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:48,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T04:54:48,144 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T04:54:48,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:48,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:48,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T04:54:48,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T04:54:48,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:48,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:48,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T04:54:48,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T04:54:48,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:48,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:48,146 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T04:54:48,147 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:48,148 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:48,149 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T04:54:48,149 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T04:54:48,149 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T04:54:48,150 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T04:54:48,151 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72264223, jitterRate=0.07682083547115326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T04:54:48,151 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T04:54:48,153 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732510488141Writing region info on filesystem at 1732510488141Initializing all the Stores at 1732510488141Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510488141Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510488142 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510488142Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510488142Cleaning up temporary data from old regions at 1732510488149 (+7 ms)Running coprocessor post-open hooks at 1732510488151 (+2 ms)Region opened successfully at 1732510488153 (+2 ms) 2024-11-25T04:54:48,158 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732510488087 2024-11-25T04:54:48,167 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T04:54:48,167 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T04:54:48,168 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=13aa81add8b8,38993,1732510485653 2024-11-25T04:54:48,169 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 13aa81add8b8,38993,1732510485653, state=OPEN 2024-11-25T04:54:48,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T04:54:48,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T04:54:48,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T04:54:48,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:48,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:48,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:48,200 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=13aa81add8b8,38993,1732510485653 2024-11-25T04:54:48,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T04:54:48,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=13aa81add8b8,38993,1732510485653 in 277 msec 2024-11-25T04:54:48,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T04:54:48,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 819 msec 2024-11-25T04:54:48,212 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:48,212 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T04:54:48,229 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T04:54:48,231 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=13aa81add8b8,38993,1732510485653, seqNum=-1] 2024-11-25T04:54:48,246 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T04:54:48,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46541, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T04:54:48,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1220 sec 2024-11-25T04:54:48,266 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732510488266, completionTime=-1 2024-11-25T04:54:48,269 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T04:54:48,269 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T04:54:48,291 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T04:54:48,291 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732510548291 2024-11-25T04:54:48,291 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732510608291 2024-11-25T04:54:48,291 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-25T04:54:48,293 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T04:54:48,298 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43575,1732510484520-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:48,299 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43575,1732510484520-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:48,299 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43575,1732510484520-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:48,300 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-13aa81add8b8:43575, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:48,300 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:48,301 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:48,307 DEBUG [master/13aa81add8b8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T04:54:48,326 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.477sec 2024-11-25T04:54:48,328 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T04:54:48,329 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T04:54:48,330 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T04:54:48,331 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T04:54:48,331 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T04:54:48,332 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43575,1732510484520-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T04:54:48,333 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43575,1732510484520-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T04:54:48,334 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T04:54:48,335 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T04:54:48,335 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43575,1732510484520-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:48,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T04:54:48,355 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T04:54:48,355 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T04:54:48,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 13aa81add8b8,43575,-1 for getting cluster id 2024-11-25T04:54:48,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T04:54:48,371 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1be64eb-6480-4100-976f-2d3d9f204c71' 2024-11-25T04:54:48,373 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T04:54:48,374 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1be64eb-6480-4100-976f-2d3d9f204c71" 2024-11-25T04:54:48,376 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ef9d6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T04:54:48,376 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [13aa81add8b8,43575,-1] 2024-11-25T04:54:48,379 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T04:54:48,382 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:48,383 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T04:54:48,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T04:54:48,387 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T04:54:48,394 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=13aa81add8b8,38993,1732510485653, seqNum=-1] 2024-11-25T04:54:48,395 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T04:54:48,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T04:54:48,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=13aa81add8b8,43575,1732510484520 2024-11-25T04:54:48,434 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T04:54:48,443 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 13aa81add8b8,43575,1732510484520 2024-11-25T04:54:48,446 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2742ae79 2024-11-25T04:54:48,447 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T04:54:48,449 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T04:54:48,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732510488429', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T04:54:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 2024-11-25T04:54:48,466 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T04:54:48,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732510488429" procId is: 4 2024-11-25T04:54:48,469 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:48,470 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T04:54:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:48,487 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9583427ea97ce6cf6b631fdf0c3f2597, NAME => 'testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732510488429', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:48,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:48,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(1722): Closing 9583427ea97ce6cf6b631fdf0c3f2597, disabling compactions & flushes 2024-11-25T04:54:48,494 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:48,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:48,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. after waiting 0 ms 2024-11-25T04:54:48,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:48,495 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:48,495 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732510488429-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9583427ea97ce6cf6b631fdf0c3f2597: Waiting for close lock at 1732510488494Disabling compacts and flushes for region at 1732510488494Disabling writes for close at 1732510488494Writing region close event to WAL at 1732510488494Closed at 1732510488494 2024-11-25T04:54:48,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T04:54:48,501 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732510488496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732510488496"}]},"ts":"1732510488496"} 2024-11-25T04:54:48,505 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T04:54:48,507 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T04:54:48,509 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732510488429","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732510488507"}]},"ts":"1732510488507"} 2024-11-25T04:54:48,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732510488429, state=ENABLING in hbase:meta 2024-11-25T04:54:48,514 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {13aa81add8b8=0} racks are {/default-rack=0} 2024-11-25T04:54:48,515 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T04:54:48,515 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T04:54:48,515 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T04:54:48,515 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T04:54:48,515 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T04:54:48,515 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T04:54:48,515 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T04:54:48,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, ASSIGN}] 2024-11-25T04:54:48,518 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, ASSIGN 2024-11-25T04:54:48,519 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, ASSIGN; state=OFFLINE, location=13aa81add8b8,32787,1732510485458; forceNewPlan=false, retain=false 2024-11-25T04:54:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:48,677 INFO [13aa81add8b8:43575 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T04:54:48,679 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9583427ea97ce6cf6b631fdf0c3f2597, regionState=OPENING, regionLocation=13aa81add8b8,32787,1732510485458 2024-11-25T04:54:48,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, ASSIGN because future has completed 2024-11-25T04:54:48,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,32787,1732510485458}] 2024-11-25T04:54:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:48,836 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T04:54:48,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59533, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T04:54:48,842 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:48,843 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9583427ea97ce6cf6b631fdf0c3f2597, NAME => 'testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:48,843 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732510488429 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,844 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:48,844 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,844 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,845 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,850 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9583427ea97ce6cf6b631fdf0c3f2597 columnFamilyName family 2024-11-25T04:54:48,850 DEBUG [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:48,851 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.HStore(327): Store=9583427ea97ce6cf6b631fdf0c3f2597/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:48,852 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,853 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,853 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,854 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,854 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,855 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,861 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T04:54:48,862 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9583427ea97ce6cf6b631fdf0c3f2597; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73041282, jitterRate=0.08839991688728333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T04:54:48,862 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:48,863 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9583427ea97ce6cf6b631fdf0c3f2597: Running coprocessor pre-open hook at 1732510488844Writing region info on filesystem at 1732510488844Initializing all the Stores at 1732510488845 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510488845Cleaning up temporary data from old regions at 1732510488854 (+9 ms)Running coprocessor post-open hooks at 1732510488862 (+8 ms)Region opened successfully at 1732510488863 (+1 ms) 2024-11-25T04:54:48,864 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597., pid=6, masterSystemTime=1732510488836 2024-11-25T04:54:48,867 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:48,867 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:48,867 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9583427ea97ce6cf6b631fdf0c3f2597, regionState=OPEN, openSeqNum=2, regionLocation=13aa81add8b8,32787,1732510485458 2024-11-25T04:54:48,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,32787,1732510485458 because future has completed 2024-11-25T04:54:48,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T04:54:48,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,32787,1732510485458 in 189 msec 2024-11-25T04:54:48,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T04:54:48,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, ASSIGN in 358 msec 2024-11-25T04:54:48,877 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T04:54:48,878 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732510488429","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732510488877"}]},"ts":"1732510488877"} 2024-11-25T04:54:48,880 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732510488429, state=ENABLED in hbase:meta 2024-11-25T04:54:48,881 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T04:54:48,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732510488429 in 422 msec 2024-11-25T04:54:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:49,102 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732510488429 get assigned. Timeout = 60000ms 2024-11-25T04:54:49,102 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732510488429 completed 2024-11-25T04:54:49,105 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T04:54:49,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732510488429 assigned to meta. Checking AM states. 2024-11-25T04:54:49,117 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T04:54:49,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732510488429 assigned. 2024-11-25T04:54:49,128 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732510488429', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597., hostname=13aa81add8b8,32787,1732510485458, seqNum=2] 2024-11-25T04:54:49,130 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T04:54:49,132 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T04:54:49,141 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9583427ea97ce6cf6b631fdf0c3f2597 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T04:54:49,162 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:49,195 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/.tmp/family/ddc721957fc94e65b28d7cd9f389b48a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T04:54:49,200 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/.tmp/family/ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:49,231 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/.tmp/family/ddc721957fc94e65b28d7cd9f389b48a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:49,233 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a, delay=983 ms 2024-11-25T04:54:49,235 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T04:54:49,241 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9583427ea97ce6cf6b631fdf0c3f2597 in 99ms, sequenceid=6, compaction requested=false 2024-11-25T04:54:49,242 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732510488429' 2024-11-25T04:54:49,243 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9583427ea97ce6cf6b631fdf0c3f2597: 2024-11-25T04:54:50,221 DEBUG [hfile-prefetch-1732510489233 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ddc721957fc94e65b28d7cd9f389b48a_0. 2024-11-25T04:54:50,226 DEBUG [hfile-prefetch-1732510489233 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ddc721957fc94e65b28d7cd9f389b48a_117. 2024-11-25T04:54:50,228 DEBUG [hfile-prefetch-1732510489233 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a, with total blocks 2, and data blocks 1 2024-11-25T04:54:50,228 DEBUG [hfile-prefetch-1732510489233 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T04:54:50,229 DEBUG [hfile-prefetch-1732510489233 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T04:54:50,229 DEBUG [hfile-prefetch-1732510489233 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a. Will sleep 100ms and try the verification again. 2024-11-25T04:54:50,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9583427ea97ce6cf6b631fdf0c3f2597, source=13aa81add8b8,32787,1732510485458, destination=13aa81add8b8,38993,1732510485653, warming up region on 13aa81add8b8,38993,1732510485653 2024-11-25T04:54:50,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9583427ea97ce6cf6b631fdf0c3f2597, source=13aa81add8b8,32787,1732510485458, destination=13aa81add8b8,38993,1732510485653, running balancer 2024-11-25T04:54:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, REOPEN/MOVE 2024-11-25T04:54:50,262 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, REOPEN/MOVE 2024-11-25T04:54:50,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9583427ea97ce6cf6b631fdf0c3f2597, NAME => 'testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:50,263 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,264 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9583427ea97ce6cf6b631fdf0c3f2597, regionState=CLOSING, regionLocation=13aa81add8b8,32787,1732510485458 2024-11-25T04:54:50,264 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9583427ea97ce6cf6b631fdf0c3f2597 columnFamilyName family 2024-11-25T04:54:50,265 DEBUG [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:50,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, REOPEN/MOVE because future has completed 2024-11-25T04:54:50,268 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-25T04:54:50,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,32787,1732510485458}] 2024-11-25T04:54:50,269 DEBUG [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:50,269 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.HStore(327): Store=9583427ea97ce6cf6b631fdf0c3f2597/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(1722): Closing 9583427ea97ce6cf6b631fdf0c3f2597, disabling compactions & flushes 2024-11-25T04:54:50,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. after waiting 0 ms 2024-11-25T04:54:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,271 DEBUG [StoreFileCloser-9583427ea97ce6cf6b631fdf0c3f2597-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:50,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(1676): Region close journal for 9583427ea97ce6cf6b631fdf0c3f2597: Waiting for close lock at 1732510490269Disabling compacts and flushes for region at 1732510490269Disabling writes for close at 1732510490269Writing region close event to WAL at 1732510490271 (+2 ms)Closed at 1732510490271 2024-11-25T04:54:50,272 WARN [hfile-prefetch-1732510489233 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-25T04:54:50,273 DEBUG [hfile-prefetch-1732510489233 {}] hfile.PrefetchExecutor(121): Prefetch completed for ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43575 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T04:54:50,380 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T04:54:50,380 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T04:54:50,381 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:50,381 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:50,397 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T04:54:50,397 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T04:54:50,397 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:50,397 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:50,397 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-25T04:54:50,397 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732510490380 2024-11-25T04:54:50,397 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T04:54:50,399 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T04:54:50,399 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732510490397 2024-11-25T04:54:50,400 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T04:54:50,426 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,426 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-25T04:54:50,427 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9583427ea97ce6cf6b631fdf0c3f2597, disabling compactions & flushes 2024-11-25T04:54:50,427 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,427 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,427 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. after waiting 0 ms 2024-11-25T04:54:50,427 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,428 DEBUG [StoreFileCloser-9583427ea97ce6cf6b631fdf0c3f2597-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file ddc721957fc94e65b28d7cd9f389b48a, starting offset: 0, end offset: 9223372036854775807 2024-11-25T04:54:50,429 DEBUG [StoreFileCloser-9583427ea97ce6cf6b631fdf0c3f2597-family-1 {}] bucket.BucketCache(856): removed key ddc721957fc94e65b28d7cd9f389b48a_0 from back map with offset lock 2094080 in the evict process 2024-11-25T04:54:50,434 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T04:54:50,435 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,436 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9583427ea97ce6cf6b631fdf0c3f2597: Waiting for close lock at 1732510490427Running coprocessor pre-close hooks at 1732510490427Disabling compacts and flushes for region at 1732510490427Disabling writes for close at 1732510490427Writing region close event to WAL at 1732510490430 (+3 ms)Running coprocessor post-close hooks at 1732510490434 (+4 ms)Closed at 1732510490435 (+1 ms) 2024-11-25T04:54:50,436 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9583427ea97ce6cf6b631fdf0c3f2597 move to 13aa81add8b8,38993,1732510485653 record at close sequenceid=6 2024-11-25T04:54:50,440 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,440 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9583427ea97ce6cf6b631fdf0c3f2597, regionState=CLOSED 2024-11-25T04:54:50,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,32787,1732510485458 because future has completed 2024-11-25T04:54:50,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T04:54:50,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,32787,1732510485458 in 177 msec 2024-11-25T04:54:50,448 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, REOPEN/MOVE; state=CLOSED, location=13aa81add8b8,38993,1732510485653; forceNewPlan=false, retain=false 2024-11-25T04:54:50,498 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T04:54:50,498 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T04:54:50,498 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:50,498 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:50,504 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T04:54:50,504 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732510490498 2024-11-25T04:54:50,504 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T04:54:50,598 INFO [13aa81add8b8:43575 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T04:54:50,599 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9583427ea97ce6cf6b631fdf0c3f2597, regionState=OPENING, regionLocation=13aa81add8b8,38993,1732510485653 2024-11-25T04:54:50,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, REOPEN/MOVE because future has completed 2024-11-25T04:54:50,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,38993,1732510485653}] 2024-11-25T04:54:50,762 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,762 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9583427ea97ce6cf6b631fdf0c3f2597, NAME => 'testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:50,763 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732510488429 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,763 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:50,763 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,763 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,765 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,766 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9583427ea97ce6cf6b631fdf0c3f2597 columnFamilyName family 2024-11-25T04:54:50,766 DEBUG [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:50,768 DEBUG [StoreFileOpener-9583427ea97ce6cf6b631fdf0c3f2597-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a, delay=953 ms 2024-11-25T04:54:50,770 DEBUG [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:50,770 INFO [StoreOpener-9583427ea97ce6cf6b631fdf0c3f2597-1 {}] regionserver.HStore(327): Store=9583427ea97ce6cf6b631fdf0c3f2597/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:50,771 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,772 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,772 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,773 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,773 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,778 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,779 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9583427ea97ce6cf6b631fdf0c3f2597; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73407420, jitterRate=0.09385579824447632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T04:54:50,779 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,780 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9583427ea97ce6cf6b631fdf0c3f2597: Running coprocessor pre-open hook at 1732510490764Writing region info on filesystem at 1732510490764Initializing all the Stores at 1732510490764Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510490764Cleaning up temporary data from old regions at 1732510490773 (+9 ms)Running coprocessor post-open hooks at 1732510490779 (+6 ms)Region opened successfully at 1732510490780 (+1 ms) 2024-11-25T04:54:50,781 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597., pid=9, masterSystemTime=1732510490759 2024-11-25T04:54:50,784 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,784 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9583427ea97ce6cf6b631fdf0c3f2597, regionState=OPEN, openSeqNum=10, regionLocation=13aa81add8b8,38993,1732510485653 2024-11-25T04:54:50,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,38993,1732510485653 because future has completed 2024-11-25T04:54:50,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T04:54:50,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9583427ea97ce6cf6b631fdf0c3f2597, server=13aa81add8b8,38993,1732510485653 in 184 msec 2024-11-25T04:54:50,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732510488429, region=9583427ea97ce6cf6b631fdf0c3f2597, REOPEN/MOVE in 530 msec 2024-11-25T04:54:50,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T04:54:50,815 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T04:54:50,815 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:50,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:50,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:50,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T04:54:50,821 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T04:54:50,821 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=68825266, stopped=false 2024-11-25T04:54:50,822 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=13aa81add8b8,43575,1732510484520 2024-11-25T04:54:50,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:50,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:50,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:50,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:50,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:50,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:50,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T04:54:50,849 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T04:54:50,850 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:50,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:50,850 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:50,851 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '13aa81add8b8,32787,1732510485458' ***** 2024-11-25T04:54:50,851 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:50,851 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T04:54:50,851 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '13aa81add8b8,38993,1732510485653' ***** 2024-11-25T04:54:50,851 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:50,851 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T04:54:50,851 INFO [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T04:54:50,851 INFO [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T04:54:50,852 INFO [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.persistence 2024-11-25T04:54:50,852 INFO [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/bucket.persistence 2024-11-25T04:54:50,852 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T04:54:50,852 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T04:54:50,852 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T04:54:50,852 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T04:54:50,852 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T04:54:50,853 INFO [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T04:54:50,853 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T04:54:50,853 INFO [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T04:54:50,853 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T04:54:50,853 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T04:54:50,863 DEBUG [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1487): Thread RS:1;13aa81add8b8:38993 started persisting bucket cache to file 2024-11-25T04:54:50,863 DEBUG [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:50,863 DEBUG [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:50,863 DEBUG [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1487): Thread RS:0;13aa81add8b8:32787 started persisting bucket cache to file 2024-11-25T04:54:50,863 DEBUG [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:50,863 DEBUG [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:50,866 DEBUG [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T04:54:50,866 DEBUG [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T04:54:50,866 DEBUG [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732510490863 2024-11-25T04:54:50,866 DEBUG [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732510490863 2024-11-25T04:54:50,866 DEBUG [RS:0;13aa81add8b8:32787 {}] bucket.BucketCache(1509): Thread RS:0;13aa81add8b8:32787 finished persisting bucket cache to file, renaming 2024-11-25T04:54:50,866 DEBUG [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1509): Thread RS:1;13aa81add8b8:38993 finished persisting bucket cache to file, renaming 2024-11-25T04:54:50,866 WARN [RS:1;13aa81add8b8:38993 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-25T04:54:50,866 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T04:54:50,866 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T04:54:50,866 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T04:54:50,866 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T04:54:50,866 INFO [RS:0;13aa81add8b8:32787 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T04:54:50,866 INFO [RS:1;13aa81add8b8:38993 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T04:54:50,867 INFO [RS:1;13aa81add8b8:38993 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T04:54:50,867 INFO [RS:0;13aa81add8b8:32787 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T04:54:50,867 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(959): stopping server 13aa81add8b8,32787,1732510485458 2024-11-25T04:54:50,867 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,867 INFO [RS:0;13aa81add8b8:32787 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T04:54:50,867 INFO [RS:0;13aa81add8b8:32787 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;13aa81add8b8:32787. 2024-11-25T04:54:50,867 DEBUG [RS:0;13aa81add8b8:32787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:50,867 DEBUG [RS:0;13aa81add8b8:32787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:50,867 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(959): stopping server 13aa81add8b8,38993,1732510485653 2024-11-25T04:54:50,867 INFO [RS:1;13aa81add8b8:38993 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T04:54:50,867 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(976): stopping server 13aa81add8b8,32787,1732510485458; all regions closed. 2024-11-25T04:54:50,867 INFO [RS:1;13aa81add8b8:38993 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;13aa81add8b8:38993. 2024-11-25T04:54:50,868 DEBUG [RS:1;13aa81add8b8:38993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:50,868 DEBUG [RS:1;13aa81add8b8:38993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:50,868 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T04:54:50,868 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T04:54:50,868 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T04:54:50,868 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T04:54:50,868 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9583427ea97ce6cf6b631fdf0c3f2597, disabling compactions & flushes 2024-11-25T04:54:50,868 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T04:54:50,868 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,868 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,868 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. after waiting 0 ms 2024-11-25T04:54:50,868 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,868 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T04:54:50,869 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 9583427ea97ce6cf6b631fdf0c3f2597=testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597.} 2024-11-25T04:54:50,869 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T04:54:50,869 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T04:54:50,869 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T04:54:50,869 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T04:54:50,869 DEBUG [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9583427ea97ce6cf6b631fdf0c3f2597 2024-11-25T04:54:50,869 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T04:54:50,869 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T04:54:50,869 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-25T04:54:50,870 DEBUG [StoreFileCloser-9583427ea97ce6cf6b631fdf0c3f2597-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/family/ddc721957fc94e65b28d7cd9f389b48a 2024-11-25T04:54:50,873 DEBUG [RS:0;13aa81add8b8:32787 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs 2024-11-25T04:54:50,873 INFO [RS:0;13aa81add8b8:32787 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 13aa81add8b8%2C32787%2C1732510485458:(num 1732510487711) 2024-11-25T04:54:50,873 DEBUG [RS:0;13aa81add8b8:32787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:50,873 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:50,873 INFO [RS:0;13aa81add8b8:32787 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T04:54:50,873 INFO [RS:0;13aa81add8b8:32787 {}] hbase.ChoreService(370): Chore service for: regionserver/13aa81add8b8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T04:54:50,874 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T04:54:50,874 INFO [regionserver/13aa81add8b8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T04:54:50,874 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T04:54:50,874 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T04:54:50,874 INFO [RS:0;13aa81add8b8:32787 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T04:54:50,874 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnRegionMove1732510488429/9583427ea97ce6cf6b631fdf0c3f2597/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T04:54:50,874 INFO [RS:0;13aa81add8b8:32787 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32787 2024-11-25T04:54:50,874 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,875 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9583427ea97ce6cf6b631fdf0c3f2597: Waiting for close lock at 1732510490867Running coprocessor pre-close hooks at 1732510490868 (+1 ms)Disabling compacts and flushes for region at 1732510490868Disabling writes for close at 1732510490868Writing region close event to WAL at 1732510490870 (+2 ms)Running coprocessor post-close hooks at 1732510490874 (+4 ms)Closed at 1732510490874 2024-11-25T04:54:50,875 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597. 2024-11-25T04:54:50,879 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3ff12830c2834b7e9f1e8dd1feb45280 2024-11-25T04:54:50,895 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/info/3ff12830c2834b7e9f1e8dd1feb45280 is 205, key is testBlockEvictionOnRegionMove1732510488429,,1732510488451.9583427ea97ce6cf6b631fdf0c3f2597./info:regioninfo/1732510490785/Put/seqid=0 2024-11-25T04:54:50,896 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/info/3ff12830c2834b7e9f1e8dd1feb45280 2024-11-25T04:54:50,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/13aa81add8b8,32787,1732510485458 2024-11-25T04:54:50,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:50,898 INFO [RS:0;13aa81add8b8:32787 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T04:54:50,904 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 00ca30dc2fe84727b9295b9a5acdb638 2024-11-25T04:54:50,910 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [13aa81add8b8,32787,1732510485458] 2024-11-25T04:54:50,917 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/ns/00ca30dc2fe84727b9295b9a5acdb638 is 43, key is default/ns:d/1732510488251/Put/seqid=0 2024-11-25T04:54:50,917 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/ns/00ca30dc2fe84727b9295b9a5acdb638 2024-11-25T04:54:50,918 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/13aa81add8b8,32787,1732510485458 already deleted, retry=false 2024-11-25T04:54:50,918 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 13aa81add8b8,32787,1732510485458 expired; onlineServers=1 2024-11-25T04:54:50,926 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 84fcc4e1794a43b38fcc04117124f647 2024-11-25T04:54:50,937 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/table/84fcc4e1794a43b38fcc04117124f647 is 78, key is testBlockEvictionOnRegionMove1732510488429/table:state/1732510488877/Put/seqid=0 2024-11-25T04:54:50,938 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/table/84fcc4e1794a43b38fcc04117124f647 2024-11-25T04:54:50,940 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/info/3ff12830c2834b7e9f1e8dd1feb45280 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/3ff12830c2834b7e9f1e8dd1feb45280 2024-11-25T04:54:50,941 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/3ff12830c2834b7e9f1e8dd1feb45280, delay=943 ms 2024-11-25T04:54:50,942 INFO [regionserver/13aa81add8b8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:50,942 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/3ff12830c2834b7e9f1e8dd1feb45280, entries=15, sequenceid=15, filesize=7.6 K 2024-11-25T04:54:50,943 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/ns/00ca30dc2fe84727b9295b9a5acdb638 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/00ca30dc2fe84727b9295b9a5acdb638 2024-11-25T04:54:50,944 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/00ca30dc2fe84727b9295b9a5acdb638, delay=982 ms 2024-11-25T04:54:50,945 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/00ca30dc2fe84727b9295b9a5acdb638, entries=2, sequenceid=15, filesize=5.0 K 2024-11-25T04:54:50,945 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/table/84fcc4e1794a43b38fcc04117124f647 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/84fcc4e1794a43b38fcc04117124f647 2024-11-25T04:54:50,946 INFO [regionserver/13aa81add8b8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:50,946 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/84fcc4e1794a43b38fcc04117124f647, delay=918 ms 2024-11-25T04:54:50,948 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/84fcc4e1794a43b38fcc04117124f647, entries=2, sequenceid=15, filesize=5.3 K 2024-11-25T04:54:50,948 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false 2024-11-25T04:54:50,948 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T04:54:50,950 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/3ff12830c2834b7e9f1e8dd1feb45280 2024-11-25T04:54:50,950 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/00ca30dc2fe84727b9295b9a5acdb638 2024-11-25T04:54:50,951 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/84fcc4e1794a43b38fcc04117124f647 2024-11-25T04:54:50,955 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-25T04:54:50,956 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T04:54:50,956 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T04:54:50,956 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732510490868Running coprocessor pre-close hooks at 1732510490869 (+1 ms)Disabling compacts and flushes for region at 1732510490869Disabling writes for close at 1732510490869Obtaining lock to block concurrent updates at 1732510490869Preparing flush snapshotting stores in 1588230740 at 1732510490869Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732510490870 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732510490870Flushing 1588230740/info: creating writer at 1732510490870Flushing 1588230740/info: appending metadata at 1732510490894 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732510490894Flushing 1588230740/ns: creating writer at 1732510490898 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732510490916 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732510490916Flushing 1588230740/table: creating writer at 1732510490920 (+4 ms)Flushing 1588230740/table: appending metadata at 1732510490937 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732510490937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40b1cc0e: reopening flushed file at 1732510490940 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@624f4783: reopening flushed file at 1732510490943 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f9ee511: reopening flushed file at 1732510490945 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false at 1732510490948 (+3 ms)Writing region close event to WAL at 1732510490951 (+3 ms)Running coprocessor post-close hooks at 1732510490955 (+4 ms)Closed at 1732510490956 (+1 ms) 2024-11-25T04:54:50,956 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T04:54:51,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:51,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32787-0x10171a199bd0001, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:51,010 INFO [RS:0;13aa81add8b8:32787 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T04:54:51,011 INFO [RS:0;13aa81add8b8:32787 {}] regionserver.HRegionServer(1031): Exiting; stopping=13aa81add8b8,32787,1732510485458; zookeeper connection closed. 2024-11-25T04:54:51,011 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4af7727 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4af7727 2024-11-25T04:54:51,069 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(976): stopping server 13aa81add8b8,38993,1732510485653; all regions closed. 2024-11-25T04:54:51,073 DEBUG [RS:1;13aa81add8b8:38993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs 2024-11-25T04:54:51,073 INFO [RS:1;13aa81add8b8:38993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 13aa81add8b8%2C38993%2C1732510485653.meta:.meta(num 1732510488112) 2024-11-25T04:54:51,077 DEBUG [RS:1;13aa81add8b8:38993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs 2024-11-25T04:54:51,077 INFO [RS:1;13aa81add8b8:38993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 13aa81add8b8%2C38993%2C1732510485653:(num 1732510487711) 2024-11-25T04:54:51,077 DEBUG [RS:1;13aa81add8b8:38993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:51,077 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:51,078 INFO [RS:1;13aa81add8b8:38993 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T04:54:51,078 INFO [RS:1;13aa81add8b8:38993 {}] hbase.ChoreService(370): Chore service for: regionserver/13aa81add8b8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T04:54:51,078 INFO [RS:1;13aa81add8b8:38993 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T04:54:51,078 INFO [regionserver/13aa81add8b8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T04:54:51,079 INFO [RS:1;13aa81add8b8:38993 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38993 2024-11-25T04:54:51,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:51,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/13aa81add8b8,38993,1732510485653 2024-11-25T04:54:51,101 INFO [RS:1;13aa81add8b8:38993 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T04:54:51,102 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [13aa81add8b8,38993,1732510485653] 2024-11-25T04:54:51,118 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/13aa81add8b8,38993,1732510485653 already deleted, retry=false 2024-11-25T04:54:51,118 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 13aa81add8b8,38993,1732510485653 expired; onlineServers=0 2024-11-25T04:54:51,118 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '13aa81add8b8,43575,1732510484520' ***** 2024-11-25T04:54:51,118 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T04:54:51,118 INFO [M:0;13aa81add8b8:43575 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T04:54:51,118 INFO [M:0;13aa81add8b8:43575 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T04:54:51,118 DEBUG [M:0;13aa81add8b8:43575 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T04:54:51,119 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T04:54:51,119 DEBUG [M:0;13aa81add8b8:43575 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T04:54:51,119 DEBUG [master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.small.0-1732510487286 {}] cleaner.HFileCleaner(306): Exit Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.small.0-1732510487286,5,FailOnTimeoutGroup] 2024-11-25T04:54:51,119 DEBUG [master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.large.0-1732510487285 {}] cleaner.HFileCleaner(306): Exit Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.large.0-1732510487285,5,FailOnTimeoutGroup] 2024-11-25T04:54:51,119 INFO [M:0;13aa81add8b8:43575 {}] hbase.ChoreService(370): Chore service for: master/13aa81add8b8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T04:54:51,119 INFO [M:0;13aa81add8b8:43575 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T04:54:51,119 DEBUG [M:0;13aa81add8b8:43575 {}] master.HMaster(1795): Stopping service threads 2024-11-25T04:54:51,119 INFO [M:0;13aa81add8b8:43575 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T04:54:51,119 INFO [M:0;13aa81add8b8:43575 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T04:54:51,120 INFO [M:0;13aa81add8b8:43575 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T04:54:51,120 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T04:54:51,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T04:54:51,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,126 DEBUG [M:0;13aa81add8b8:43575 {}] zookeeper.ZKUtil(347): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T04:54:51,126 WARN [M:0;13aa81add8b8:43575 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T04:54:51,127 INFO [M:0;13aa81add8b8:43575 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.lastflushedseqids 2024-11-25T04:54:51,137 INFO [M:0;13aa81add8b8:43575 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T04:54:51,137 INFO [M:0;13aa81add8b8:43575 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T04:54:51,137 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T04:54:51,137 INFO [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,137 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,137 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T04:54:51,137 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,137 INFO [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-25T04:54:51,146 DEBUG [M:0;13aa81add8b8:43575 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d2a82d0d7b6145fc85f82ec58269e3e3 2024-11-25T04:54:51,158 DEBUG [M:0;13aa81add8b8:43575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2a82d0d7b6145fc85f82ec58269e3e3 is 82, key is hbase:meta,,1/info:regioninfo/1732510488167/Put/seqid=0 2024-11-25T04:54:51,159 INFO [M:0;13aa81add8b8:43575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2a82d0d7b6145fc85f82ec58269e3e3 2024-11-25T04:54:51,166 DEBUG [M:0;13aa81add8b8:43575 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9792694ec3d545c5a007d67352a99599 2024-11-25T04:54:51,178 DEBUG [M:0;13aa81add8b8:43575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9792694ec3d545c5a007d67352a99599 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732510488882/Put/seqid=0 2024-11-25T04:54:51,179 INFO [M:0;13aa81add8b8:43575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9792694ec3d545c5a007d67352a99599 2024-11-25T04:54:51,186 DEBUG [M:0;13aa81add8b8:43575 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e4a29b01b0b3474689f56208eaa1f750 2024-11-25T04:54:51,198 DEBUG [M:0;13aa81add8b8:43575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4a29b01b0b3474689f56208eaa1f750 is 69, key is 13aa81add8b8,32787,1732510485458/rs:state/1732510487338/Put/seqid=0 2024-11-25T04:54:51,199 INFO [M:0;13aa81add8b8:43575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4a29b01b0b3474689f56208eaa1f750 2024-11-25T04:54:51,201 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2a82d0d7b6145fc85f82ec58269e3e3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d2a82d0d7b6145fc85f82ec58269e3e3 2024-11-25T04:54:51,202 INFO [M:0;13aa81add8b8:43575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d2a82d0d7b6145fc85f82ec58269e3e3, entries=8, sequenceid=84, filesize=5.5 K 2024-11-25T04:54:51,203 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9792694ec3d545c5a007d67352a99599 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9792694ec3d545c5a007d67352a99599 2024-11-25T04:54:51,204 INFO [M:0;13aa81add8b8:43575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9792694ec3d545c5a007d67352a99599, entries=9, sequenceid=84, filesize=6.5 K 2024-11-25T04:54:51,205 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4a29b01b0b3474689f56208eaa1f750 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e4a29b01b0b3474689f56208eaa1f750 2024-11-25T04:54:51,206 INFO [M:0;13aa81add8b8:43575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e4a29b01b0b3474689f56208eaa1f750, entries=2, sequenceid=84, filesize=5.1 K 2024-11-25T04:54:51,207 INFO [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false 2024-11-25T04:54:51,209 INFO [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,209 DEBUG [M:0;13aa81add8b8:43575 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732510491137Disabling compacts and flushes for region at 1732510491137Disabling writes for close at 1732510491137Obtaining lock to block concurrent updates at 1732510491137Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732510491137Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732510491138 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732510491138Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732510491138Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732510491157 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732510491157Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732510491160 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732510491178 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732510491178Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732510491181 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732510491198 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732510491198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13ef1e16: reopening flushed file at 1732510491200 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f4d4dd: reopening flushed file at 1732510491202 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2786735: reopening flushed file at 1732510491204 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false at 1732510491207 (+3 ms)Writing region close event to WAL at 1732510491208 (+1 ms)Closed at 1732510491208 2024-11-25T04:54:51,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:51,210 INFO [RS:1;13aa81add8b8:38993 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T04:54:51,210 INFO [RS:1;13aa81add8b8:38993 {}] regionserver.HRegionServer(1031): Exiting; stopping=13aa81add8b8,38993,1732510485653; zookeeper connection closed. 2024-11-25T04:54:51,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x10171a199bd0002, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:51,210 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@450fc50a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@450fc50a 2024-11-25T04:54:51,210 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T04:54:51,210 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-25T04:54:51,210 INFO [M:0;13aa81add8b8:43575 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T04:54:51,211 INFO [M:0;13aa81add8b8:43575 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43575 2024-11-25T04:54:51,211 INFO [M:0;13aa81add8b8:43575 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T04:54:51,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:51,327 INFO [M:0;13aa81add8b8:43575 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T04:54:51,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43575-0x10171a199bd0000, quorum=127.0.0.1:49566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:51,334 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T04:54:51,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T04:54:51,353 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,361 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/13aa81add8b8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/13aa81add8b8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732510490769 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732510490944 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/13aa81add8b8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/13aa81add8b8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732510490941 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732510489233 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 479), ProcessCount=11 (was 11), AvailableMemoryMB=11740 (was 12211) 2024-11-25T04:54:51,366 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=11740 2024-11-25T04:54:51,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/hadoop-log-dir so I do NOT create it in target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206 2024-11-25T04:54:51,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/hadoop-tmp-dir so I do NOT create it in target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206 2024-11-25T04:54:51,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,369 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/cluster_d5b8a981-ca2b-497e-8164-847a190422b6, deleteOnExit=true 2024-11-25T04:54:51,371 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/cluster_d5b8a981-ca2b-497e-8164-847a190422b6/zookeeper_0, clientPort=60588, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/cluster_d5b8a981-ca2b-497e-8164-847a190422b6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/cluster_d5b8a981-ca2b-497e-8164-847a190422b6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T04:54:51,372 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60588 2024-11-25T04:54:51,383 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f with version=8 2024-11-25T04:54:51,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/hbase-staging 2024-11-25T04:54:51,385 INFO [Time-limited test {}] client.ConnectionUtils(128): master/13aa81add8b8:0 server-side Connection retries=45 2024-11-25T04:54:51,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,386 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T04:54:51,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T04:54:51,386 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T04:54:51,386 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T04:54:51,387 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42577 2024-11-25T04:54:51,389 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42577 connecting to ZooKeeper ensemble=127.0.0.1:60588 2024-11-25T04:54:51,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:425770x0, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T04:54:51,426 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42577-0x10171a1b5b00000 connected 2024-11-25T04:54:51,501 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,502 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:51,504 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f, hbase.cluster.distributed=false 2024-11-25T04:54:51,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T04:54:51,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42577 2024-11-25T04:54:51,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42577 2024-11-25T04:54:51,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42577 2024-11-25T04:54:51,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42577 2024-11-25T04:54:51,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42577 2024-11-25T04:54:51,526 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/13aa81add8b8:0 server-side Connection retries=45 2024-11-25T04:54:51,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,526 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T04:54:51,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T04:54:51,527 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T04:54:51,527 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T04:54:51,527 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43719 2024-11-25T04:54:51,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43719 connecting to ZooKeeper ensemble=127.0.0.1:60588 2024-11-25T04:54:51,529 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,529 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:437190x0, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T04:54:51,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43719-0x10171a1b5b00001 connected 2024-11-25T04:54:51,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:51,540 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T04:54:51,540 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.cache 2024-11-25T04:54:51,540 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T04:54:51,540 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T04:54:51,540 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T04:54:51,541 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T04:54:51,541 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T04:54:51,541 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T04:54:51,541 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T04:54:51,541 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T04:54:51,541 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence is complete. 2024-11-25T04:54:51,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T04:54:51,544 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T04:54:51,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43719 2024-11-25T04:54:51,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43719 2024-11-25T04:54:51,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43719 2024-11-25T04:54:51,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43719 2024-11-25T04:54:51,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43719 2024-11-25T04:54:51,564 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/13aa81add8b8:0 server-side Connection retries=45 2024-11-25T04:54:51,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,564 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T04:54:51,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:51,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T04:54:51,564 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T04:54:51,565 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T04:54:51,565 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46879 2024-11-25T04:54:51,567 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46879 connecting to ZooKeeper ensemble=127.0.0.1:60588 2024-11-25T04:54:51,567 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,567 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468790x0, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T04:54:51,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468790x0, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:51,577 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46879-0x10171a1b5b00002 connected 2024-11-25T04:54:51,577 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T04:54:51,577 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.cache 2024-11-25T04:54:51,577 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T04:54:51,578 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T04:54:51,578 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T04:54:51,578 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T04:54:51,578 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T04:54:51,578 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T04:54:51,578 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T04:54:51,578 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T04:54:51,578 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence is complete. 2024-11-25T04:54:51,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T04:54:51,580 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T04:54:51,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46879 2024-11-25T04:54:51,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46879 2024-11-25T04:54:51,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46879 2024-11-25T04:54:51,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46879 2024-11-25T04:54:51,585 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46879 2024-11-25T04:54:51,600 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;13aa81add8b8:42577 2024-11-25T04:54:51,602 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/13aa81add8b8,42577,1732510491385 2024-11-25T04:54:51,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:51,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:51,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:51,610 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/13aa81add8b8,42577,1732510491385 2024-11-25T04:54:51,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T04:54:51,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T04:54:51,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,618 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T04:54:51,619 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/13aa81add8b8,42577,1732510491385 from backup master directory 2024-11-25T04:54:51,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:51,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:51,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/13aa81add8b8,42577,1732510491385 2024-11-25T04:54:51,626 WARN [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T04:54:51,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T04:54:51,626 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=13aa81add8b8,42577,1732510491385 2024-11-25T04:54:51,627 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/hbase.id] with ID: e6404bdd-9df6-4768-95b3-72a534fa72a9 2024-11-25T04:54:51,627 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.tmp/hbase.id 2024-11-25T04:54:51,631 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/hbase.id] 2024-11-25T04:54:51,661 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:51,661 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T04:54:51,665 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 4ms. 2024-11-25T04:54:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,682 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T04:54:51,683 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T04:54:51,684 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:51,708 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store 2024-11-25T04:54:51,717 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:51,717 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T04:54:51,718 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,718 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,718 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T04:54:51,718 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,718 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:51,718 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732510491717Disabling compacts and flushes for region at 1732510491717Disabling writes for close at 1732510491718 (+1 ms)Writing region close event to WAL at 1732510491718Closed at 1732510491718 2024-11-25T04:54:51,718 WARN [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/.initializing 2024-11-25T04:54:51,719 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/WALs/13aa81add8b8,42577,1732510491385 2024-11-25T04:54:51,726 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T04:54:51,729 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=13aa81add8b8%2C42577%2C1732510491385, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/WALs/13aa81add8b8,42577,1732510491385, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/oldWALs, maxLogs=10 2024-11-25T04:54:51,745 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/WALs/13aa81add8b8,42577,1732510491385/13aa81add8b8%2C42577%2C1732510491385.1732510491730 2024-11-25T04:54:51,745 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:51,746 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:51,746 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:51,746 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,746 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T04:54:51,751 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:51,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T04:54:51,755 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:51,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T04:54:51,759 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:51,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T04:54:51,763 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:51,764 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,764 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,764 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,764 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,764 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,765 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T04:54:51,765 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T04:54:51,770 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T04:54:51,770 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66152350, jitterRate=-0.014253169298171997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T04:54:51,771 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732510491746Initializing all the Stores at 1732510491746Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510491746Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510491747 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510491747Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510491747Cleaning up temporary data from old regions at 1732510491764 (+17 ms)Region opened successfully at 1732510491771 (+7 ms) 2024-11-25T04:54:51,772 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T04:54:51,775 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff2393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=13aa81add8b8/172.17.0.2:0 2024-11-25T04:54:51,776 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T04:54:51,777 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T04:54:51,777 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T04:54:51,777 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T04:54:51,777 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T04:54:51,778 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-25T04:54:51,778 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T04:54:51,779 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T04:54:51,780 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T04:54:51,797 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T04:54:51,797 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T04:54:51,798 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T04:54:51,809 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T04:54:51,810 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T04:54:51,811 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T04:54:51,822 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T04:54:51,823 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T04:54:51,834 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T04:54:51,837 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T04:54:51,847 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T04:54:51,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:51,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:51,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:51,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,857 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=13aa81add8b8,42577,1732510491385, sessionid=0x10171a1b5b00000, setting cluster-up flag (Was=false) 2024-11-25T04:54:51,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,901 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T04:54:51,903 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=13aa81add8b8,42577,1732510491385 2024-11-25T04:54:51,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:51,947 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T04:54:51,949 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=13aa81add8b8,42577,1732510491385 2024-11-25T04:54:51,950 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T04:54:51,951 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:51,952 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T04:54:51,952 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T04:54:51,952 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 13aa81add8b8,42577,1732510491385 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T04:54:51,954 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:51,954 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:51,954 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:51,955 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/13aa81add8b8:0, corePoolSize=5, maxPoolSize=5 2024-11-25T04:54:51,955 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/13aa81add8b8:0, corePoolSize=10, maxPoolSize=10 2024-11-25T04:54:51,955 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:51,955 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/13aa81add8b8:0, corePoolSize=2, maxPoolSize=2 2024-11-25T04:54:51,955 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:51,956 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732510521956 2024-11-25T04:54:51,957 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T04:54:51,957 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T04:54:51,957 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:51,957 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T04:54:51,957 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T04:54:51,957 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T04:54:51,957 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T04:54:51,957 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T04:54:51,958 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:51,958 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,958 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T04:54:51,958 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T04:54:51,958 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T04:54:51,958 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T04:54:51,959 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T04:54:51,959 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T04:54:51,959 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.large.0-1732510491959,5,FailOnTimeoutGroup] 2024-11-25T04:54:51,959 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.small.0-1732510491959,5,FailOnTimeoutGroup] 2024-11-25T04:54:51,960 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:51,960 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T04:54:51,960 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:51,960 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:51,972 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T04:54:51,972 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:51,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:51,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T04:54:51,982 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T04:54:51,982 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:51,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T04:54:51,986 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T04:54:51,986 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:51,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T04:54:51,988 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(746): ClusterId : e6404bdd-9df6-4768-95b3-72a534fa72a9 2024-11-25T04:54:51,988 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(746): ClusterId : e6404bdd-9df6-4768-95b3-72a534fa72a9 2024-11-25T04:54:51,988 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T04:54:51,988 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T04:54:51,990 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T04:54:51,990 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:51,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T04:54:51,993 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T04:54:51,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:51,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:51,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T04:54:51,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:51,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:51,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T04:54:51,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T04:54:51,995 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T04:54:51,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T04:54:51,998 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T04:54:51,998 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T04:54:51,998 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T04:54:51,998 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T04:54:52,001 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T04:54:52,001 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65606431, jitterRate=-0.022387996315956116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T04:54:52,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732510491978Initializing all the Stores at 1732510491978Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510491978Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510491979 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510491979Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510491979Cleaning up temporary data from old regions at 1732510491994 (+15 ms)Region opened successfully at 1732510492001 (+7 ms) 2024-11-25T04:54:52,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T04:54:52,002 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T04:54:52,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T04:54:52,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T04:54:52,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T04:54:52,003 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T04:54:52,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732510492002Disabling compacts and flushes for region at 1732510492002Disabling writes for close at 1732510492002Writing region close event to WAL at 1732510492002Closed at 1732510492002 2024-11-25T04:54:52,003 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:52,004 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T04:54:52,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T04:54:52,005 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T04:54:52,005 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T04:54:52,010 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T04:54:52,010 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T04:54:52,011 DEBUG [RS:0;13aa81add8b8:43719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251d4ee5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=13aa81add8b8/172.17.0.2:0 2024-11-25T04:54:52,011 DEBUG [RS:1;13aa81add8b8:46879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec6d6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=13aa81add8b8/172.17.0.2:0 2024-11-25T04:54:52,023 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;13aa81add8b8:43719 2024-11-25T04:54:52,023 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T04:54:52,023 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T04:54:52,023 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T04:54:52,024 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;13aa81add8b8:46879 2024-11-25T04:54:52,024 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(2659): reportForDuty to master=13aa81add8b8,42577,1732510491385 with port=43719, startcode=1732510491526 2024-11-25T04:54:52,024 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T04:54:52,024 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T04:54:52,024 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T04:54:52,025 DEBUG [RS:0;13aa81add8b8:43719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T04:54:52,025 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(2659): reportForDuty to master=13aa81add8b8,42577,1732510491385 with port=46879, startcode=1732510491563 2024-11-25T04:54:52,025 DEBUG [RS:1;13aa81add8b8:46879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T04:54:52,027 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55441, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T04:54:52,028 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58717, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T04:54:52,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42577 {}] master.ServerManager(517): Registering regionserver=13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,030 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 13aa81add8b8,46879,1732510491563 2024-11-25T04:54:52,030 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42577 {}] master.ServerManager(517): Registering regionserver=13aa81add8b8,46879,1732510491563 2024-11-25T04:54:52,030 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:52,030 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T04:54:52,030 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T04:54:52,031 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:52,031 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T04:54:52,032 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T04:54:52,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:52,067 DEBUG [RS:0;13aa81add8b8:43719 {}] zookeeper.ZKUtil(111): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,067 WARN [RS:0;13aa81add8b8:43719 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T04:54:52,067 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [13aa81add8b8,43719,1732510491526] 2024-11-25T04:54:52,067 INFO [RS:0;13aa81add8b8:43719 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:52,067 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [13aa81add8b8,46879,1732510491563] 2024-11-25T04:54:52,068 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,068 DEBUG [RS:1;13aa81add8b8:46879 {}] zookeeper.ZKUtil(111): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/13aa81add8b8,46879,1732510491563 2024-11-25T04:54:52,068 WARN [RS:1;13aa81add8b8:46879 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T04:54:52,068 INFO [RS:1;13aa81add8b8:46879 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:52,068 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,46879,1732510491563 2024-11-25T04:54:52,075 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T04:54:52,078 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T04:54:52,078 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T04:54:52,079 INFO [RS:0;13aa81add8b8:43719 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T04:54:52,079 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,079 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T04:54:52,080 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T04:54:52,080 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,080 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T04:54:52,081 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,081 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,081 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,081 INFO [RS:1;13aa81add8b8:46879 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T04:54:52,081 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,081 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,081 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,081 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/13aa81add8b8:0, corePoolSize=2, maxPoolSize=2 2024-11-25T04:54:52,082 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,082 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,082 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T04:54:52,082 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,082 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,082 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,082 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,082 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T04:54:52,082 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:52,083 DEBUG [RS:0;13aa81add8b8:43719 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:52,083 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,083 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,083 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,083 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,083 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,083 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,083 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/13aa81add8b8:0, corePoolSize=2, maxPoolSize=2 2024-11-25T04:54:52,084 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,084 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,084 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,084 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,084 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,084 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:52,084 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:52,085 DEBUG [RS:1;13aa81add8b8:46879 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:52,085 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43719,1732510491526-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T04:54:52,085 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,086 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,086 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,46879,1732510491563-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T04:54:52,101 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T04:54:52,101 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,46879,1732510491563-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,101 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,101 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.Replication(171): 13aa81add8b8,46879,1732510491563 started 2024-11-25T04:54:52,102 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T04:54:52,102 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,43719,1732510491526-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,102 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,102 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.Replication(171): 13aa81add8b8,43719,1732510491526 started 2024-11-25T04:54:52,115 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,115 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,115 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1482): Serving as 13aa81add8b8,46879,1732510491563, RpcServer on 13aa81add8b8/172.17.0.2:46879, sessionid=0x10171a1b5b00002 2024-11-25T04:54:52,115 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1482): Serving as 13aa81add8b8,43719,1732510491526, RpcServer on 13aa81add8b8/172.17.0.2:43719, sessionid=0x10171a1b5b00001 2024-11-25T04:54:52,115 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T04:54:52,115 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T04:54:52,115 DEBUG [RS:1;13aa81add8b8:46879 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 13aa81add8b8,46879,1732510491563 2024-11-25T04:54:52,115 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,46879,1732510491563' 2024-11-25T04:54:52,115 DEBUG [RS:0;13aa81add8b8:43719 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,115 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T04:54:52,115 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,43719,1732510491526' 2024-11-25T04:54:52,115 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T04:54:52,116 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T04:54:52,116 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T04:54:52,117 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T04:54:52,117 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T04:54:52,117 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T04:54:52,117 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T04:54:52,117 DEBUG [RS:1;13aa81add8b8:46879 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 13aa81add8b8,46879,1732510491563 2024-11-25T04:54:52,117 DEBUG [RS:0;13aa81add8b8:43719 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,117 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,46879,1732510491563' 2024-11-25T04:54:52,117 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,43719,1732510491526' 2024-11-25T04:54:52,117 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T04:54:52,117 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T04:54:52,117 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T04:54:52,118 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T04:54:52,118 DEBUG [RS:1;13aa81add8b8:46879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T04:54:52,118 INFO [RS:1;13aa81add8b8:46879 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T04:54:52,118 DEBUG [RS:0;13aa81add8b8:43719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T04:54:52,118 INFO [RS:1;13aa81add8b8:46879 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T04:54:52,118 INFO [RS:0;13aa81add8b8:43719 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T04:54:52,118 INFO [RS:0;13aa81add8b8:43719 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T04:54:52,156 WARN [13aa81add8b8:42577 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T04:54:52,219 INFO [RS:0;13aa81add8b8:43719 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T04:54:52,219 INFO [RS:1;13aa81add8b8:46879 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T04:54:52,221 INFO [RS:0;13aa81add8b8:43719 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=13aa81add8b8%2C43719%2C1732510491526, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,43719,1732510491526, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs, maxLogs=55 2024-11-25T04:54:52,221 INFO [RS:1;13aa81add8b8:46879 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=13aa81add8b8%2C46879%2C1732510491563, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,46879,1732510491563, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs, maxLogs=55 2024-11-25T04:54:52,242 INFO [RS:1;13aa81add8b8:46879 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,46879,1732510491563/13aa81add8b8%2C46879%2C1732510491563.1732510492225 2024-11-25T04:54:52,242 INFO [RS:0;13aa81add8b8:43719 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,43719,1732510491526/13aa81add8b8%2C43719%2C1732510491526.1732510492226 2024-11-25T04:54:52,243 DEBUG [RS:1;13aa81add8b8:46879 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:52,243 DEBUG [RS:0;13aa81add8b8:43719 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:52,406 DEBUG [13aa81add8b8:42577 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T04:54:52,407 DEBUG [13aa81add8b8:42577 {}] balancer.BalancerClusterState(204): Hosts are {13aa81add8b8=0} racks are {/default-rack=0} 2024-11-25T04:54:52,408 DEBUG [13aa81add8b8:42577 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T04:54:52,408 DEBUG [13aa81add8b8:42577 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T04:54:52,409 DEBUG [13aa81add8b8:42577 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T04:54:52,409 DEBUG [13aa81add8b8:42577 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T04:54:52,409 INFO [13aa81add8b8:42577 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T04:54:52,409 INFO [13aa81add8b8:42577 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T04:54:52,409 DEBUG [13aa81add8b8:42577 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T04:54:52,410 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,411 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 13aa81add8b8,43719,1732510491526, state=OPENING 2024-11-25T04:54:52,426 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T04:54:52,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:52,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:52,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:52,435 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T04:54:52,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:52,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:52,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:52,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=13aa81add8b8,43719,1732510491526}] 2024-11-25T04:54:52,588 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T04:54:52,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47903, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T04:54:52,594 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T04:54:52,594 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:52,595 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T04:54:52,597 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=13aa81add8b8%2C43719%2C1732510491526.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,43719,1732510491526, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs, maxLogs=55 2024-11-25T04:54:52,615 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,43719,1732510491526/13aa81add8b8%2C43719%2C1732510491526.meta.1732510492598.meta 2024-11-25T04:54:52,615 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:52,616 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:52,616 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T04:54:52,616 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T04:54:52,616 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T04:54:52,616 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T04:54:52,616 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:52,617 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T04:54:52,617 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T04:54:52,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T04:54:52,618 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T04:54:52,618 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:52,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:52,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T04:54:52,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T04:54:52,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:52,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:52,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T04:54:52,620 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T04:54:52,620 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:52,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:52,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T04:54:52,620 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T04:54:52,620 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:52,621 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T04:54:52,621 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T04:54:52,621 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:52,622 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740 2024-11-25T04:54:52,622 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T04:54:52,622 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T04:54:52,622 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T04:54:52,623 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T04:54:52,623 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65398695, jitterRate=-0.025483503937721252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T04:54:52,623 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T04:54:52,624 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732510492617Writing region info on filesystem at 1732510492617Initializing all the Stores at 1732510492617Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510492617Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510492617Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510492617Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732510492617Cleaning up temporary data from old regions at 1732510492622 (+5 ms)Running coprocessor post-open hooks at 1732510492623 (+1 ms)Region opened successfully at 1732510492624 (+1 ms) 2024-11-25T04:54:52,625 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732510492587 2024-11-25T04:54:52,627 DEBUG [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T04:54:52,627 INFO [RS_OPEN_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T04:54:52,627 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,628 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 13aa81add8b8,43719,1732510491526, state=OPEN 2024-11-25T04:54:52,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T04:54:52,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T04:54:52,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T04:54:52,647 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=13aa81add8b8,43719,1732510491526 2024-11-25T04:54:52,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:52,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:52,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T04:54:52,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T04:54:52,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=13aa81add8b8,43719,1732510491526 in 212 msec 2024-11-25T04:54:52,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T04:54:52,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 645 msec 2024-11-25T04:54:52,652 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T04:54:52,652 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T04:54:52,655 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T04:54:52,655 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=13aa81add8b8,43719,1732510491526, seqNum=-1] 2024-11-25T04:54:52,655 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T04:54:52,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T04:54:52,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 711 msec 2024-11-25T04:54:52,663 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732510492663, completionTime=-1 2024-11-25T04:54:52,663 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T04:54:52,663 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T04:54:52,665 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T04:54:52,665 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732510552665 2024-11-25T04:54:52,665 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732510612665 2024-11-25T04:54:52,665 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-25T04:54:52,666 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T04:54:52,666 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,42577,1732510491385-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,666 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,42577,1732510491385-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,666 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,42577,1732510491385-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,666 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-13aa81add8b8:42577, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,666 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,667 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,670 DEBUG [master/13aa81add8b8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.047sec 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,42577,1732510491385-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T04:54:52,673 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,42577,1732510491385-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T04:54:52,674 DEBUG [master/13aa81add8b8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T04:54:52,674 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T04:54:52,674 INFO [master/13aa81add8b8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,42577,1732510491385-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:52,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T04:54:52,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 13aa81add8b8,42577,-1 for getting cluster id 2024-11-25T04:54:52,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T04:54:52,691 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e6404bdd-9df6-4768-95b3-72a534fa72a9' 2024-11-25T04:54:52,691 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T04:54:52,691 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e6404bdd-9df6-4768-95b3-72a534fa72a9" 2024-11-25T04:54:52,692 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d6690f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T04:54:52,692 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [13aa81add8b8,42577,-1] 2024-11-25T04:54:52,692 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T04:54:52,693 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:52,694 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T04:54:52,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T04:54:52,696 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T04:54:52,697 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=13aa81add8b8,43719,1732510491526, seqNum=-1] 2024-11-25T04:54:52,698 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T04:54:52,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T04:54:52,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=13aa81add8b8,42577,1732510491385 2024-11-25T04:54:52,702 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T04:54:52,703 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 13aa81add8b8,42577,1732510491385 2024-11-25T04:54:52,703 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fee78e 2024-11-25T04:54:52,704 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T04:54:52,706 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T04:54:52,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732510492702', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T04:54:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42577 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 2024-11-25T04:54:52,709 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T04:54:52,709 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:52,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732510492702" procId is: 4 2024-11-25T04:54:52,710 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T04:54:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:52,720 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 833a8ff51b5eafbf15d5cad082c9dfdf, NAME => 'testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732510492702', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:52,726 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:52,726 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(1722): Closing 833a8ff51b5eafbf15d5cad082c9dfdf, disabling compactions & flushes 2024-11-25T04:54:52,726 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:52,726 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:52,726 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. after waiting 0 ms 2024-11-25T04:54:52,726 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:52,726 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:52,726 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732510492702-pool-0 {}] regionserver.HRegion(1676): Region close journal for 833a8ff51b5eafbf15d5cad082c9dfdf: Waiting for close lock at 1732510492726Disabling compacts and flushes for region at 1732510492726Disabling writes for close at 1732510492726Writing region close event to WAL at 1732510492726Closed at 1732510492726 2024-11-25T04:54:52,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T04:54:52,727 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732510492727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732510492727"}]},"ts":"1732510492727"} 2024-11-25T04:54:52,730 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T04:54:52,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T04:54:52,731 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732510492702","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732510492731"}]},"ts":"1732510492731"} 2024-11-25T04:54:52,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732510492702, state=ENABLING in hbase:meta 2024-11-25T04:54:52,734 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {13aa81add8b8=0} racks are {/default-rack=0} 2024-11-25T04:54:52,735 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T04:54:52,735 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T04:54:52,735 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T04:54:52,735 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T04:54:52,735 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T04:54:52,735 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T04:54:52,735 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T04:54:52,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN}] 2024-11-25T04:54:52,736 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN 2024-11-25T04:54:52,737 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN; state=OFFLINE, location=13aa81add8b8,46879,1732510491563; forceNewPlan=false, retain=false 2024-11-25T04:54:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:52,888 INFO [13aa81add8b8:42577 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T04:54:52,889 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=833a8ff51b5eafbf15d5cad082c9dfdf, regionState=OPENING, regionLocation=13aa81add8b8,46879,1732510491563 2024-11-25T04:54:52,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN because future has completed 2024-11-25T04:54:52,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 833a8ff51b5eafbf15d5cad082c9dfdf, server=13aa81add8b8,46879,1732510491563}] 2024-11-25T04:54:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:53,049 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T04:54:53,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33531, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T04:54:53,054 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:53,054 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 833a8ff51b5eafbf15d5cad082c9dfdf, NAME => 'testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:53,055 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732510492702 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,055 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:53,055 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,055 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,057 INFO [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,062 INFO [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 833a8ff51b5eafbf15d5cad082c9dfdf columnFamilyName family 2024-11-25T04:54:53,062 DEBUG [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:53,062 INFO [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] regionserver.HStore(327): Store=833a8ff51b5eafbf15d5cad082c9dfdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:53,062 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,063 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,063 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,063 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,063 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,064 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,075 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T04:54:53,075 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 833a8ff51b5eafbf15d5cad082c9dfdf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72568064, jitterRate=0.08134841918945312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T04:54:53,075 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:53,076 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 833a8ff51b5eafbf15d5cad082c9dfdf: Running coprocessor pre-open hook at 1732510493056Writing region info on filesystem at 1732510493056Initializing all the Stores at 1732510493056Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510493056Cleaning up temporary data from old regions at 1732510493063 (+7 ms)Running coprocessor post-open hooks at 1732510493075 (+12 ms)Region opened successfully at 1732510493076 (+1 ms) 2024-11-25T04:54:53,076 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf., pid=6, masterSystemTime=1732510493049 2024-11-25T04:54:53,079 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:53,079 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:53,079 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=833a8ff51b5eafbf15d5cad082c9dfdf, regionState=OPEN, openSeqNum=2, regionLocation=13aa81add8b8,46879,1732510491563 2024-11-25T04:54:53,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 833a8ff51b5eafbf15d5cad082c9dfdf, server=13aa81add8b8,46879,1732510491563 because future has completed 2024-11-25T04:54:53,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T04:54:53,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 833a8ff51b5eafbf15d5cad082c9dfdf, server=13aa81add8b8,46879,1732510491563 in 188 msec 2024-11-25T04:54:53,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T04:54:53,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN in 350 msec 2024-11-25T04:54:53,088 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T04:54:53,088 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732510492702","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732510493088"}]},"ts":"1732510493088"} 2024-11-25T04:54:53,090 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732510492702, state=ENABLED in hbase:meta 2024-11-25T04:54:53,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T04:54:53,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732510492702 in 384 msec 2024-11-25T04:54:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T04:54:53,343 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732510492702 completed 2024-11-25T04:54:53,343 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732510492702 get assigned. Timeout = 60000ms 2024-11-25T04:54:53,344 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T04:54:53,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732510492702 assigned to meta. Checking AM states. 2024-11-25T04:54:53,350 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T04:54:53,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732510492702 assigned. 2024-11-25T04:54:53,355 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732510492702', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf., hostname=13aa81add8b8,46879,1732510491563, seqNum=2] 2024-11-25T04:54:53,355 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T04:54:53,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T04:54:53,364 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 833a8ff51b5eafbf15d5cad082c9dfdf 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T04:54:53,377 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b996a64503f24827bcf3eb51b56eb6ce 2024-11-25T04:54:53,390 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/.tmp/family/b996a64503f24827bcf3eb51b56eb6ce is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T04:54:53,391 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/.tmp/family/b996a64503f24827bcf3eb51b56eb6ce 2024-11-25T04:54:53,394 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/.tmp/family/b996a64503f24827bcf3eb51b56eb6ce as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce 2024-11-25T04:54:53,395 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce, delay=976 ms 2024-11-25T04:54:53,397 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T04:54:53,398 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 833a8ff51b5eafbf15d5cad082c9dfdf in 34ms, sequenceid=6, compaction requested=false 2024-11-25T04:54:53,398 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732510492702' 2024-11-25T04:54:53,399 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 833a8ff51b5eafbf15d5cad082c9dfdf: 2024-11-25T04:54:53,643 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T04:54:54,373 DEBUG [hfile-prefetch-1732510490769 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b996a64503f24827bcf3eb51b56eb6ce_0. 2024-11-25T04:54:54,375 DEBUG [hfile-prefetch-1732510490769 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b996a64503f24827bcf3eb51b56eb6ce_117. 2024-11-25T04:54:54,375 DEBUG [hfile-prefetch-1732510490769 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce, with total blocks 2, and data blocks 1 2024-11-25T04:54:54,376 DEBUG [hfile-prefetch-1732510490769 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T04:54:54,376 DEBUG [hfile-prefetch-1732510490769 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T04:54:54,376 DEBUG [hfile-prefetch-1732510490769 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce. Will sleep 100ms and try the verification again. 2024-11-25T04:54:54,399 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;13aa81add8b8:46879,5,FailOnTimeoutGroup] 2024-11-25T04:54:54,399 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '13aa81add8b8,46879,1732510491563' ***** 2024-11-25T04:54:54,399 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-25T04:54:54,400 INFO [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T04:54:54,450 INFO [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence 2024-11-25T04:54:54,450 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T04:54:54,450 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T04:54:54,450 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T04:54:54,451 INFO [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T04:54:54,451 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T04:54:54,461 DEBUG [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1487): Thread RS:1;13aa81add8b8:46879 started persisting bucket cache to file 2024-11-25T04:54:54,461 DEBUG [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:54,461 DEBUG [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:54,465 DEBUG [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-25T04:54:54,465 DEBUG [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732510494461 2024-11-25T04:54:54,465 DEBUG [RS:1;13aa81add8b8:46879 {}] bucket.BucketCache(1509): Thread RS:1;13aa81add8b8:46879 finished persisting bucket cache to file, renaming 2024-11-25T04:54:54,465 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T04:54:54,465 INFO [RS:1;13aa81add8b8:46879 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T04:54:54,465 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T04:54:54,465 INFO [RS:1;13aa81add8b8:46879 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T04:54:54,465 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(3091): Received CLOSE for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:54,466 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(959): stopping server 13aa81add8b8,46879,1732510491563 2024-11-25T04:54:54,466 INFO [RS:1;13aa81add8b8:46879 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T04:54:54,466 INFO [RS:1;13aa81add8b8:46879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;13aa81add8b8:46879. 2024-11-25T04:54:54,466 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T04:54:54,466 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 833a8ff51b5eafbf15d5cad082c9dfdf, disabling compactions & flushes 2024-11-25T04:54:54,466 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:54,466 DEBUG [RS:1;13aa81add8b8:46879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:54,466 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:54,466 DEBUG [RS:1;13aa81add8b8:46879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:54,466 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. after waiting 0 ms 2024-11-25T04:54:54,466 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:54,466 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T04:54:54,466 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1325): Online Regions={833a8ff51b5eafbf15d5cad082c9dfdf=testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.} 2024-11-25T04:54:54,467 DEBUG [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1351): Waiting on 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:54,467 DEBUG [StoreFileCloser-833a8ff51b5eafbf15d5cad082c9dfdf-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce 2024-11-25T04:54:54,467 WARN [hfile-prefetch-1732510490769 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-25T04:54:54,468 DEBUG [hfile-prefetch-1732510490769 {}] hfile.PrefetchExecutor(121): Prefetch completed for b996a64503f24827bcf3eb51b56eb6ce 2024-11-25T04:54:54,473 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T04:54:54,473 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:54,473 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 833a8ff51b5eafbf15d5cad082c9dfdf: Waiting for close lock at 1732510494466Running coprocessor pre-close hooks at 1732510494466Disabling compacts and flushes for region at 1732510494466Disabling writes for close at 1732510494466Writing region close event to WAL at 1732510494467 (+1 ms)Running coprocessor post-close hooks at 1732510494473 (+6 ms)Closed at 1732510494473 2024-11-25T04:54:54,473 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:54,489 INFO [regionserver/13aa81add8b8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:54,543 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T04:54:54,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T04:54:54,544 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:54,544 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:54,546 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T04:54:54,546 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732510494543 2024-11-25T04:54:54,546 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T04:54:54,667 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(976): stopping server 13aa81add8b8,46879,1732510491563; all regions closed. 2024-11-25T04:54:54,671 DEBUG [RS:1;13aa81add8b8:46879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs 2024-11-25T04:54:54,671 INFO [RS:1;13aa81add8b8:46879 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 13aa81add8b8%2C46879%2C1732510491563:(num 1732510492225) 2024-11-25T04:54:54,672 DEBUG [RS:1;13aa81add8b8:46879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:54,672 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:54,672 INFO [RS:1;13aa81add8b8:46879 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T04:54:54,673 INFO [RS:1;13aa81add8b8:46879 {}] hbase.ChoreService(370): Chore service for: regionserver/13aa81add8b8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T04:54:54,673 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T04:54:54,673 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T04:54:54,673 INFO [regionserver/13aa81add8b8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T04:54:54,673 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T04:54:54,674 INFO [RS:1;13aa81add8b8:46879 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T04:54:54,674 INFO [RS:1;13aa81add8b8:46879 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46879 2024-11-25T04:54:54,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/13aa81add8b8,46879,1732510491563 2024-11-25T04:54:54,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:54,705 INFO [RS:1;13aa81add8b8:46879 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T04:54:54,714 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [13aa81add8b8,46879,1732510491563] 2024-11-25T04:54:54,725 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/13aa81add8b8,46879,1732510491563 already deleted, retry=false 2024-11-25T04:54:54,726 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 13aa81add8b8,46879,1732510491563 on 13aa81add8b8,42577,1732510491385 2024-11-25T04:54:54,734 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 13aa81add8b8,46879,1732510491563, splitWal=true, meta=false 2024-11-25T04:54:54,736 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 13aa81add8b8,46879,1732510491563, splitWal=true, meta=false 2024-11-25T04:54:54,736 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 13aa81add8b8,46879,1732510491563 (carryingMeta=false) 13aa81add8b8,46879,1732510491563/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3246ea3a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T04:54:54,738 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 13aa81add8b8,46879,1732510491563 had 1 regions 2024-11-25T04:54:54,739 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 13aa81add8b8,46879,1732510491563, splitWal=true, meta=false, isMeta: false 2024-11-25T04:54:54,739 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,46879,1732510491563-splitting 2024-11-25T04:54:54,740 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,46879,1732510491563-splitting dir is empty, no logs to split. 2024-11-25T04:54:54,740 INFO [PEWorker-2 {}] master.SplitWALManager(105): 13aa81add8b8,46879,1732510491563 WAL count=0, meta=false 2024-11-25T04:54:54,742 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,46879,1732510491563-splitting dir is empty, no logs to split. 2024-11-25T04:54:54,742 INFO [PEWorker-2 {}] master.SplitWALManager(105): 13aa81add8b8,46879,1732510491563 WAL count=0, meta=false 2024-11-25T04:54:54,742 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 13aa81add8b8,46879,1732510491563 WAL splitting is done? wals=0, meta=false 2024-11-25T04:54:54,742 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 13aa81add8b8,46879,1732510491563 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,46879,1732510491563-splitting does not exist 2024-11-25T04:54:54,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN}] 2024-11-25T04:54:54,746 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN 2024-11-25T04:54:54,747 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T04:54:54,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:54,814 INFO [RS:1;13aa81add8b8:46879 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T04:54:54,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46879-0x10171a1b5b00002, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:54,814 INFO [RS:1;13aa81add8b8:46879 {}] regionserver.HRegionServer(1031): Exiting; stopping=13aa81add8b8,46879,1732510491563; zookeeper connection closed. 2024-11-25T04:54:54,814 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46e114ad {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46e114ad 2024-11-25T04:54:54,898 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=833a8ff51b5eafbf15d5cad082c9dfdf, regionState=OPENING, regionLocation=13aa81add8b8,43719,1732510491526 2024-11-25T04:54:54,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN because future has completed 2024-11-25T04:54:54,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 833a8ff51b5eafbf15d5cad082c9dfdf, server=13aa81add8b8,43719,1732510491526}] 2024-11-25T04:54:54,918 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/13aa81add8b8:0 server-side Connection retries=45 2024-11-25T04:54:54,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:54,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:54,918 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T04:54:54,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T04:54:54,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T04:54:54,918 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T04:54:54,918 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T04:54:54,919 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36693 2024-11-25T04:54:54,920 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36693 connecting to ZooKeeper ensemble=127.0.0.1:60588 2024-11-25T04:54:54,920 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:54,920 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:54,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366930x0, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T04:54:54,951 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36693-0x10171a1b5b00003 connected 2024-11-25T04:54:54,951 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-25T04:54:54,951 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-25T04:54:54,952 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T04:54:54,952 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.cache 2024-11-25T04:54:54,953 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T04:54:54,953 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T04:54:54,953 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T04:54:54,953 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T04:54:54,953 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T04:54:54,953 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T04:54:54,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T04:54:54,954 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-25T04:54:54,955 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-25T04:54:54,956 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T04:54:54,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36693 2024-11-25T04:54:54,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36693 2024-11-25T04:54:54,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36693 2024-11-25T04:54:54,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36693 2024-11-25T04:54:54,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36693 2024-11-25T04:54:54,958 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(746): ClusterId : e6404bdd-9df6-4768-95b3-72a534fa72a9 2024-11-25T04:54:54,959 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T04:54:54,965 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-25T04:54:54,965 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T04:54:54,965 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-25T04:54:54,965 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence is complete. 2024-11-25T04:54:54,968 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T04:54:54,968 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T04:54:54,976 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T04:54:54,976 DEBUG [RS:2;13aa81add8b8:36693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76365e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=13aa81add8b8/172.17.0.2:0 2024-11-25T04:54:54,986 DEBUG [RS:2;13aa81add8b8:36693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;13aa81add8b8:36693 2024-11-25T04:54:54,986 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T04:54:54,986 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T04:54:54,986 DEBUG [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T04:54:54,987 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(2659): reportForDuty to master=13aa81add8b8,42577,1732510491385 with port=36693, startcode=1732510494917 2024-11-25T04:54:54,987 DEBUG [RS:2;13aa81add8b8:36693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T04:54:54,989 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42895, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T04:54:54,990 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 13aa81add8b8,36693,1732510494917 2024-11-25T04:54:54,990 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42577 {}] master.ServerManager(517): Registering regionserver=13aa81add8b8,36693,1732510494917 2024-11-25T04:54:54,991 DEBUG [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f 2024-11-25T04:54:54,991 DEBUG [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T04:54:54,991 DEBUG [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T04:54:54,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T04:54:54,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T04:54:55,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:55,000 DEBUG [RS:2;13aa81add8b8:36693 {}] zookeeper.ZKUtil(111): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/13aa81add8b8,36693,1732510494917 2024-11-25T04:54:55,001 WARN [RS:2;13aa81add8b8:36693 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T04:54:55,001 INFO [RS:2;13aa81add8b8:36693 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T04:54:55,001 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [13aa81add8b8,36693,1732510494917] 2024-11-25T04:54:55,001 DEBUG [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,36693,1732510494917 2024-11-25T04:54:55,001 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T04:54:55,001 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T04:54:55,001 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T04:54:55,001 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T04:54:55,005 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T04:54:55,007 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T04:54:55,008 INFO [RS:2;13aa81add8b8:36693 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T04:54:55,008 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,008 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T04:54:55,008 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T04:54:55,009 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/13aa81add8b8:0, corePoolSize=2, maxPoolSize=2 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,009 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,010 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,010 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/13aa81add8b8:0, corePoolSize=1, maxPoolSize=1 2024-11-25T04:54:55,010 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:55,010 DEBUG [RS:2;13aa81add8b8:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/13aa81add8b8:0, corePoolSize=3, maxPoolSize=3 2024-11-25T04:54:55,010 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,010 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,010 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,010 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,010 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,011 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,36693,1732510494917-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T04:54:55,024 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T04:54:55,024 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=13aa81add8b8,36693,1732510494917-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,024 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,024 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.Replication(171): 13aa81add8b8,36693,1732510494917 started 2024-11-25T04:54:55,037 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T04:54:55,037 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(1482): Serving as 13aa81add8b8,36693,1732510494917, RpcServer on 13aa81add8b8/172.17.0.2:36693, sessionid=0x10171a1b5b00003 2024-11-25T04:54:55,037 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T04:54:55,037 DEBUG [RS:2;13aa81add8b8:36693 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 13aa81add8b8,36693,1732510494917 2024-11-25T04:54:55,037 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,36693,1732510494917' 2024-11-25T04:54:55,037 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T04:54:55,038 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T04:54:55,038 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T04:54:55,038 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T04:54:55,038 DEBUG [RS:2;13aa81add8b8:36693 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 13aa81add8b8,36693,1732510494917 2024-11-25T04:54:55,038 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '13aa81add8b8,36693,1732510494917' 2024-11-25T04:54:55,038 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T04:54:55,039 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T04:54:55,040 DEBUG [RS:2;13aa81add8b8:36693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T04:54:55,040 INFO [RS:2;13aa81add8b8:36693 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T04:54:55,040 INFO [RS:2;13aa81add8b8:36693 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T04:54:55,057 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,057 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 833a8ff51b5eafbf15d5cad082c9dfdf, NAME => 'testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.', STARTKEY => '', ENDKEY => ''} 2024-11-25T04:54:55,057 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732510492702 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,057 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T04:54:55,058 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,058 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,059 INFO [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,060 INFO [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 833a8ff51b5eafbf15d5cad082c9dfdf columnFamilyName family 2024-11-25T04:54:55,060 DEBUG [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T04:54:55,062 DEBUG [StoreFileOpener-833a8ff51b5eafbf15d5cad082c9dfdf-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce, delay=922 ms 2024-11-25T04:54:55,064 DEBUG [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce 2024-11-25T04:54:55,065 INFO [StoreOpener-833a8ff51b5eafbf15d5cad082c9dfdf-1 {}] regionserver.HStore(327): Store=833a8ff51b5eafbf15d5cad082c9dfdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T04:54:55,065 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,066 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,066 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,066 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,066 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,067 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,068 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 833a8ff51b5eafbf15d5cad082c9dfdf; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67650039, jitterRate=0.008064135909080505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T04:54:55,068 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,069 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 833a8ff51b5eafbf15d5cad082c9dfdf: Running coprocessor pre-open hook at 1732510495058Writing region info on filesystem at 1732510495058Initializing all the Stores at 1732510495058Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732510495058Cleaning up temporary data from old regions at 1732510495066 (+8 ms)Running coprocessor post-open hooks at 1732510495068 (+2 ms)Region opened successfully at 1732510495069 (+1 ms) 2024-11-25T04:54:55,069 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf., pid=9, masterSystemTime=1732510495054 2024-11-25T04:54:55,072 DEBUG [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,072 INFO [RS_OPEN_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,072 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=833a8ff51b5eafbf15d5cad082c9dfdf, regionState=OPEN, openSeqNum=10, regionLocation=13aa81add8b8,43719,1732510491526 2024-11-25T04:54:55,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 833a8ff51b5eafbf15d5cad082c9dfdf, server=13aa81add8b8,43719,1732510491526 because future has completed 2024-11-25T04:54:55,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T04:54:55,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 833a8ff51b5eafbf15d5cad082c9dfdf, server=13aa81add8b8,43719,1732510491526 in 175 msec 2024-11-25T04:54:55,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T04:54:55,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732510492702, region=833a8ff51b5eafbf15d5cad082c9dfdf, ASSIGN in 334 msec 2024-11-25T04:54:55,080 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 13aa81add8b8,46879,1732510491563 after splitting done 2024-11-25T04:54:55,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 13aa81add8b8,46879,1732510491563, splitWal=true, meta=false in 353 msec 2024-11-25T04:54:55,141 INFO [RS:2;13aa81add8b8:36693 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T04:54:55,143 INFO [RS:2;13aa81add8b8:36693 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=13aa81add8b8%2C36693%2C1732510494917, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,36693,1732510494917, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs, maxLogs=55 2024-11-25T04:54:55,163 INFO [RS:2;13aa81add8b8:36693 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/WALs/13aa81add8b8,36693,1732510494917/13aa81add8b8%2C36693%2C1732510494917.1732510495144 2024-11-25T04:54:55,163 DEBUG [RS:2;13aa81add8b8:36693 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T04:54:55,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T04:54:55,537 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T04:54:55,537 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:55,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:55,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:55,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T04:54:55,538 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T04:54:55,538 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=902814418, stopped=false 2024-11-25T04:54:55,538 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=13aa81add8b8,42577,1732510491385 2024-11-25T04:54:55,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:55,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:55,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T04:54:55,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T04:54:55,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:55,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:55,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:55,592 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T04:54:55,592 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:55,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:55,593 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '13aa81add8b8,43719,1732510491526' ***** 2024-11-25T04:54:55,593 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T04:54:55,593 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '13aa81add8b8,36693,1732510494917' ***** 2024-11-25T04:54:55,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:55,593 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T04:54:55,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:55,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T04:54:55,594 INFO [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T04:54:55,594 INFO [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T04:54:55,594 INFO [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence 2024-11-25T04:54:55,594 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T04:54:55,594 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T04:54:55,594 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T04:54:55,595 INFO [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/76d6aa9f-8ce3-f21b-74cf-5e8ca71ab206/bucket.persistence 2024-11-25T04:54:55,595 INFO [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T04:54:55,595 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T04:54:55,595 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T04:54:55,595 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T04:54:55,595 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T04:54:55,595 INFO [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T04:54:55,595 DEBUG [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1487): Thread RS:0;13aa81add8b8:43719 started persisting bucket cache to file 2024-11-25T04:54:55,595 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T04:54:55,596 DEBUG [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:55,596 DEBUG [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:55,600 DEBUG [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T04:54:55,600 DEBUG [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732510495595 2024-11-25T04:54:55,600 DEBUG [RS:0;13aa81add8b8:43719 {}] bucket.BucketCache(1509): Thread RS:0;13aa81add8b8:43719 finished persisting bucket cache to file, renaming 2024-11-25T04:54:55,600 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T04:54:55,600 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(3091): Received CLOSE for 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(959): stopping server 13aa81add8b8,43719,1732510491526 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;13aa81add8b8:43719. 2024-11-25T04:54:55,601 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T04:54:55,601 DEBUG [RS:0;13aa81add8b8:43719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:55,601 DEBUG [RS:0;13aa81add8b8:43719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:55,601 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 833a8ff51b5eafbf15d5cad082c9dfdf, disabling compactions & flushes 2024-11-25T04:54:55,601 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T04:54:55,601 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,601 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T04:54:55,602 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. after waiting 0 ms 2024-11-25T04:54:55,602 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T04:54:55,602 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,602 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T04:54:55,602 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 833a8ff51b5eafbf15d5cad082c9dfdf=testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf.} 2024-11-25T04:54:55,602 DEBUG [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 833a8ff51b5eafbf15d5cad082c9dfdf 2024-11-25T04:54:55,602 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T04:54:55,602 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T04:54:55,602 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T04:54:55,602 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T04:54:55,602 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T04:54:55,602 DEBUG [StoreFileCloser-833a8ff51b5eafbf15d5cad082c9dfdf-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/family/b996a64503f24827bcf3eb51b56eb6ce 2024-11-25T04:54:55,602 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T04:54:55,602 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-25T04:54:55,606 DEBUG [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1487): Thread RS:2;13aa81add8b8:36693 started persisting bucket cache to file 2024-11-25T04:54:55,606 DEBUG [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T04:54:55,606 DEBUG [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T04:54:55,606 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/default/testBlockEvictionOnGracefulStop1732510492702/833a8ff51b5eafbf15d5cad082c9dfdf/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T04:54:55,607 INFO [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,607 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 833a8ff51b5eafbf15d5cad082c9dfdf: Waiting for close lock at 1732510495601Running coprocessor pre-close hooks at 1732510495601Disabling compacts and flushes for region at 1732510495601Disabling writes for close at 1732510495602 (+1 ms)Writing region close event to WAL at 1732510495602Running coprocessor post-close hooks at 1732510495607 (+5 ms)Closed at 1732510495607 2024-11-25T04:54:55,607 DEBUG [RS_CLOSE_REGION-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf. 2024-11-25T04:54:55,608 DEBUG [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T04:54:55,608 DEBUG [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732510495606 2024-11-25T04:54:55,608 DEBUG [RS:2;13aa81add8b8:36693 {}] bucket.BucketCache(1509): Thread RS:2;13aa81add8b8:36693 finished persisting bucket cache to file, renaming 2024-11-25T04:54:55,609 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T04:54:55,609 INFO [RS:2;13aa81add8b8:36693 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T04:54:55,609 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T04:54:55,609 INFO [RS:2;13aa81add8b8:36693 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T04:54:55,609 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(959): stopping server 13aa81add8b8,36693,1732510494917 2024-11-25T04:54:55,609 INFO [RS:2;13aa81add8b8:36693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T04:54:55,609 INFO [RS:2;13aa81add8b8:36693 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;13aa81add8b8:36693. 2024-11-25T04:54:55,609 DEBUG [RS:2;13aa81add8b8:36693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T04:54:55,609 DEBUG [RS:2;13aa81add8b8:36693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:55,609 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(976): stopping server 13aa81add8b8,36693,1732510494917; all regions closed. 2024-11-25T04:54:55,611 INFO [regionserver/13aa81add8b8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:55,612 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7494d5a699f0420f98f8ba20593fb007 2024-11-25T04:54:55,612 DEBUG [RS:2;13aa81add8b8:36693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 13aa81add8b8%2C36693%2C1732510494917:(num 1732510495144) 2024-11-25T04:54:55,613 DEBUG [RS:2;13aa81add8b8:36693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] hbase.ChoreService(370): Chore service for: regionserver/13aa81add8b8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T04:54:55,613 INFO [regionserver/13aa81add8b8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T04:54:55,613 INFO [RS:2;13aa81add8b8:36693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36693 2024-11-25T04:54:55,625 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/info/7494d5a699f0420f98f8ba20593fb007 is 209, key is testBlockEvictionOnGracefulStop1732510492702,,1732510492706.833a8ff51b5eafbf15d5cad082c9dfdf./info:regioninfo/1732510495072/Put/seqid=0 2024-11-25T04:54:55,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/13aa81add8b8,36693,1732510494917 2024-11-25T04:54:55,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:55,625 INFO [RS:2;13aa81add8b8:36693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T04:54:55,626 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/info/7494d5a699f0420f98f8ba20593fb007 2024-11-25T04:54:55,633 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2eecc806bc0e4e358371077b1194716d 2024-11-25T04:54:55,633 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [13aa81add8b8,36693,1732510494917] 2024-11-25T04:54:55,641 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/13aa81add8b8,36693,1732510494917 already deleted, retry=false 2024-11-25T04:54:55,642 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 13aa81add8b8,36693,1732510494917 expired; onlineServers=1 2024-11-25T04:54:55,645 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/ns/2eecc806bc0e4e358371077b1194716d is 43, key is default/ns:d/1732510492658/Put/seqid=0 2024-11-25T04:54:55,646 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/ns/2eecc806bc0e4e358371077b1194716d 2024-11-25T04:54:55,653 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4c93594783ec412782b3489a90523ba2 2024-11-25T04:54:55,665 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/table/4c93594783ec412782b3489a90523ba2 is 80, key is testBlockEvictionOnGracefulStop1732510492702/table:state/1732510493088/Put/seqid=0 2024-11-25T04:54:55,666 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/table/4c93594783ec412782b3489a90523ba2 2024-11-25T04:54:55,668 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/info/7494d5a699f0420f98f8ba20593fb007 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/7494d5a699f0420f98f8ba20593fb007 2024-11-25T04:54:55,669 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/7494d5a699f0420f98f8ba20593fb007, delay=990 ms 2024-11-25T04:54:55,669 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/7494d5a699f0420f98f8ba20593fb007, entries=14, sequenceid=13, filesize=7.5 K 2024-11-25T04:54:55,670 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/ns/2eecc806bc0e4e358371077b1194716d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/2eecc806bc0e4e358371077b1194716d 2024-11-25T04:54:55,671 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/2eecc806bc0e4e358371077b1194716d, delay=967 ms 2024-11-25T04:54:55,671 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/2eecc806bc0e4e358371077b1194716d, entries=2, sequenceid=13, filesize=5.0 K 2024-11-25T04:54:55,672 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/.tmp/table/4c93594783ec412782b3489a90523ba2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/4c93594783ec412782b3489a90523ba2 2024-11-25T04:54:55,673 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/4c93594783ec412782b3489a90523ba2, delay=984 ms 2024-11-25T04:54:55,673 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/4c93594783ec412782b3489a90523ba2, entries=2, sequenceid=13, filesize=5.3 K 2024-11-25T04:54:55,674 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=13, compaction requested=false 2024-11-25T04:54:55,675 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/info/7494d5a699f0420f98f8ba20593fb007 2024-11-25T04:54:55,675 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/ns/2eecc806bc0e4e358371077b1194716d 2024-11-25T04:54:55,676 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/table/4c93594783ec412782b3489a90523ba2 2024-11-25T04:54:55,680 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-25T04:54:55,680 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T04:54:55,680 INFO [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T04:54:55,680 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732510495602Running coprocessor pre-close hooks at 1732510495602Disabling compacts and flushes for region at 1732510495602Disabling writes for close at 1732510495602Obtaining lock to block concurrent updates at 1732510495602Preparing flush snapshotting stores in 1588230740 at 1732510495602Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732510495603 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732510495603Flushing 1588230740/info: creating writer at 1732510495603Flushing 1588230740/info: appending metadata at 1732510495624 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732510495624Flushing 1588230740/ns: creating writer at 1732510495628 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732510495644 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732510495644Flushing 1588230740/table: creating writer at 1732510495648 (+4 ms)Flushing 1588230740/table: appending metadata at 1732510495665 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732510495665Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40817a1b: reopening flushed file at 1732510495667 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61c9fb9e: reopening flushed file at 1732510495670 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d6e335c: reopening flushed file at 1732510495671 (+1 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=13, compaction requested=false at 1732510495674 (+3 ms)Writing region close event to WAL at 1732510495676 (+2 ms)Running coprocessor post-close hooks at 1732510495680 (+4 ms)Closed at 1732510495680 2024-11-25T04:54:55,680 DEBUG [RS_CLOSE_META-regionserver/13aa81add8b8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T04:54:55,689 INFO [regionserver/13aa81add8b8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:55,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:55,734 INFO [RS:2;13aa81add8b8:36693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T04:54:55,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x10171a1b5b00003, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:55,734 INFO [RS:2;13aa81add8b8:36693 {}] regionserver.HRegionServer(1031): Exiting; stopping=13aa81add8b8,36693,1732510494917; zookeeper connection closed. 2024-11-25T04:54:55,734 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@476595bb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@476595bb 2024-11-25T04:54:55,802 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(976): stopping server 13aa81add8b8,43719,1732510491526; all regions closed. 2024-11-25T04:54:55,808 DEBUG [RS:0;13aa81add8b8:43719 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs 2024-11-25T04:54:55,808 INFO [RS:0;13aa81add8b8:43719 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 13aa81add8b8%2C43719%2C1732510491526.meta:.meta(num 1732510492598) 2024-11-25T04:54:55,810 DEBUG [RS:0;13aa81add8b8:43719 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/oldWALs 2024-11-25T04:54:55,810 INFO [RS:0;13aa81add8b8:43719 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 13aa81add8b8%2C43719%2C1732510491526:(num 1732510492226) 2024-11-25T04:54:55,810 DEBUG [RS:0;13aa81add8b8:43719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T04:54:55,810 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T04:54:55,811 INFO [RS:0;13aa81add8b8:43719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T04:54:55,811 INFO [RS:0;13aa81add8b8:43719 {}] hbase.ChoreService(370): Chore service for: regionserver/13aa81add8b8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T04:54:55,811 INFO [RS:0;13aa81add8b8:43719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T04:54:55,811 INFO [regionserver/13aa81add8b8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T04:54:55,812 INFO [RS:0;13aa81add8b8:43719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43719 2024-11-25T04:54:55,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/13aa81add8b8,43719,1732510491526 2024-11-25T04:54:55,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T04:54:55,838 INFO [RS:0;13aa81add8b8:43719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T04:54:55,850 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [13aa81add8b8,43719,1732510491526] 2024-11-25T04:54:55,858 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/13aa81add8b8,43719,1732510491526 already deleted, retry=false 2024-11-25T04:54:55,858 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 13aa81add8b8,43719,1732510491526 expired; onlineServers=0 2024-11-25T04:54:55,858 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '13aa81add8b8,42577,1732510491385' ***** 2024-11-25T04:54:55,858 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T04:54:55,859 INFO [M:0;13aa81add8b8:42577 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T04:54:55,859 INFO [M:0;13aa81add8b8:42577 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T04:54:55,859 DEBUG [M:0;13aa81add8b8:42577 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T04:54:55,859 DEBUG [M:0;13aa81add8b8:42577 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T04:54:55,859 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T04:54:55,859 DEBUG [master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.small.0-1732510491959 {}] cleaner.HFileCleaner(306): Exit Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.small.0-1732510491959,5,FailOnTimeoutGroup] 2024-11-25T04:54:55,859 DEBUG [master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.large.0-1732510491959 {}] cleaner.HFileCleaner(306): Exit Thread[master/13aa81add8b8:0:becomeActiveMaster-HFileCleaner.large.0-1732510491959,5,FailOnTimeoutGroup] 2024-11-25T04:54:55,859 INFO [M:0;13aa81add8b8:42577 {}] hbase.ChoreService(370): Chore service for: master/13aa81add8b8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T04:54:55,859 INFO [M:0;13aa81add8b8:42577 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T04:54:55,859 DEBUG [M:0;13aa81add8b8:42577 {}] master.HMaster(1795): Stopping service threads 2024-11-25T04:54:55,860 INFO [M:0;13aa81add8b8:42577 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T04:54:55,860 INFO [M:0;13aa81add8b8:42577 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T04:54:55,860 INFO [M:0;13aa81add8b8:42577 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T04:54:55,860 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T04:54:55,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T04:54:55,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T04:54:55,867 DEBUG [M:0;13aa81add8b8:42577 {}] zookeeper.ZKUtil(347): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T04:54:55,867 WARN [M:0;13aa81add8b8:42577 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T04:54:55,867 INFO [M:0;13aa81add8b8:42577 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/.lastflushedseqids 2024-11-25T04:54:55,873 INFO [M:0;13aa81add8b8:42577 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T04:54:55,873 INFO [M:0;13aa81add8b8:42577 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T04:54:55,874 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T04:54:55,874 INFO [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:55,874 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:55,874 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T04:54:55,874 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:55,874 INFO [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-25T04:54:55,882 DEBUG [M:0;13aa81add8b8:42577 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6194e3afe31d4d8ab3b3eb84741c91bb 2024-11-25T04:54:55,895 DEBUG [M:0;13aa81add8b8:42577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6194e3afe31d4d8ab3b3eb84741c91bb is 82, key is hbase:meta,,1/info:regioninfo/1732510492627/Put/seqid=0 2024-11-25T04:54:55,896 INFO [M:0;13aa81add8b8:42577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6194e3afe31d4d8ab3b3eb84741c91bb 2024-11-25T04:54:55,903 DEBUG [M:0;13aa81add8b8:42577 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fac4579e16d24cf79d989134a26c8a40 2024-11-25T04:54:55,915 DEBUG [M:0;13aa81add8b8:42577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fac4579e16d24cf79d989134a26c8a40 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732510493091/Put/seqid=0 2024-11-25T04:54:55,916 INFO [M:0;13aa81add8b8:42577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fac4579e16d24cf79d989134a26c8a40 2024-11-25T04:54:55,923 DEBUG [M:0;13aa81add8b8:42577 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3f56d2a7b1114879a411ba6f89ec79ce 2024-11-25T04:54:55,936 DEBUG [M:0;13aa81add8b8:42577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f56d2a7b1114879a411ba6f89ec79ce is 69, key is 13aa81add8b8,36693,1732510494917/rs:state/1732510494990/Put/seqid=0 2024-11-25T04:54:55,936 INFO [M:0;13aa81add8b8:42577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f56d2a7b1114879a411ba6f89ec79ce 2024-11-25T04:54:55,938 INFO [M:0;13aa81add8b8:42577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3f56d2a7b1114879a411ba6f89ec79ce 2024-11-25T04:54:55,939 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6194e3afe31d4d8ab3b3eb84741c91bb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6194e3afe31d4d8ab3b3eb84741c91bb 2024-11-25T04:54:55,940 INFO [M:0;13aa81add8b8:42577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6194e3afe31d4d8ab3b3eb84741c91bb, entries=8, sequenceid=83, filesize=5.5 K 2024-11-25T04:54:55,941 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fac4579e16d24cf79d989134a26c8a40 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fac4579e16d24cf79d989134a26c8a40 2024-11-25T04:54:55,942 INFO [M:0;13aa81add8b8:42577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fac4579e16d24cf79d989134a26c8a40, entries=9, sequenceid=83, filesize=6.4 K 2024-11-25T04:54:55,943 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f56d2a7b1114879a411ba6f89ec79ce as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f56d2a7b1114879a411ba6f89ec79ce 2024-11-25T04:54:55,944 INFO [M:0;13aa81add8b8:42577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3f56d2a7b1114879a411ba6f89ec79ce 2024-11-25T04:54:55,944 INFO [M:0;13aa81add8b8:42577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d8527a0-ace0-f3b7-93f1-55cdf3f3025f/c376eb8e-ce15-217e-50b0-6b7278f6048f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f56d2a7b1114879a411ba6f89ec79ce, entries=3, sequenceid=83, filesize=5.3 K 2024-11-25T04:54:55,945 INFO [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false 2024-11-25T04:54:55,946 INFO [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T04:54:55,946 DEBUG [M:0;13aa81add8b8:42577 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732510495874Disabling compacts and flushes for region at 1732510495874Disabling writes for close at 1732510495874Obtaining lock to block concurrent updates at 1732510495874Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732510495874Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1732510495874Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732510495875 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732510495875Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732510495895 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732510495895Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732510495898 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732510495915 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732510495915Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732510495918 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732510495935 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732510495935Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f6ce1f: reopening flushed file at 1732510495938 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e601293: reopening flushed file at 1732510495940 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d268649: reopening flushed file at 1732510495942 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false at 1732510495945 (+3 ms)Writing region close event to WAL at 1732510495946 (+1 ms)Closed at 1732510495946 2024-11-25T04:54:55,948 INFO [M:0;13aa81add8b8:42577 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T04:54:55,948 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T04:54:55,948 INFO [M:0;13aa81add8b8:42577 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42577 2024-11-25T04:54:55,948 INFO [M:0;13aa81add8b8:42577 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T04:54:55,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:55,950 INFO [RS:0;13aa81add8b8:43719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T04:54:55,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43719-0x10171a1b5b00001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:55,950 INFO [RS:0;13aa81add8b8:43719 {}] regionserver.HRegionServer(1031): Exiting; stopping=13aa81add8b8,43719,1732510491526; zookeeper connection closed. 2024-11-25T04:54:55,950 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a838440 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a838440 2024-11-25T04:54:55,951 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T04:54:56,075 INFO [M:0;13aa81add8b8:42577 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T04:54:56,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:56,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42577-0x10171a1b5b00000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T04:54:56,079 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T04:54:56,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T04:54:56,080 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T04:54:56,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=413 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=11644 (was 11740)