2024-11-22 16:45:01,438 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 16:45:01,455 main DEBUG Took 0.015833 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 16:45:01,456 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 16:45:01,456 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 16:45:01,457 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 16:45:01,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,467 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 16:45:01,481 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,483 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,484 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,485 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,485 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,486 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,487 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,489 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,491 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,492 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,493 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,494 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,495 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:45:01,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,497 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 16:45:01,499 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:45:01,501 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 16:45:01,503 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 16:45:01,504 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 16:45:01,505 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 16:45:01,506 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 16:45:01,517 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 16:45:01,519 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 16:45:01,521 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 16:45:01,521 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 16:45:01,522 main DEBUG createAppenders(={Console}) 2024-11-22 16:45:01,522 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-22 16:45:01,523 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 16:45:01,523 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-22 16:45:01,523 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 16:45:01,524 main DEBUG OutputStream closed 2024-11-22 16:45:01,524 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 16:45:01,524 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 16:45:01,524 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-22 16:45:01,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 16:45:01,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 16:45:01,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 16:45:01,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 16:45:01,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 16:45:01,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 16:45:01,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 16:45:01,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 16:45:01,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 16:45:01,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 16:45:01,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 16:45:01,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 16:45:01,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 16:45:01,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 16:45:01,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 16:45:01,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 16:45:01,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 16:45:01,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 16:45:01,604 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 16:45:01,604 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-22 16:45:01,604 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 16:45:01,605 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-22T16:45:01,619 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-22 16:45:01,622 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 16:45:01,622 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T16:45:01,882 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844 2024-11-22T16:45:01,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=8054 2024-11-22T16:45:01,996 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T16:45:02,085 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:02,101 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/cluster_b9d043c7-46a5-92a1-7f61-7c70b2b9bc5a, deleteOnExit=true 2024-11-22T16:45:02,193 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/cluster_b9d043c7-46a5-92a1-7f61-7c70b2b9bc5a/zookeeper_0, clientPort=62922, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/cluster_b9d043c7-46a5-92a1-7f61-7c70b2b9bc5a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/cluster_b9d043c7-46a5-92a1-7f61-7c70b2b9bc5a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T16:45:02,205 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62922 2024-11-22T16:45:02,206 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:02,298 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 with version=8 2024-11-22T16:45:02,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/hbase-staging 2024-11-22T16:45:02,402 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T16:45:02,676 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:45:02,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:02,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:02,697 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:45:02,697 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:02,698 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:45:02,838 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T16:45:02,929 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:45:02,969 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T16:45:02,979 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T16:45:02,984 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:45:03,028 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15225 (auto-detected) 2024-11-22T16:45:03,030 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T16:45:03,056 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33793 2024-11-22T16:45:03,088 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33793 connecting to ZooKeeper ensemble=127.0.0.1:62922 2024-11-22T16:45:03,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:337930x0, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:45:03,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33793-0x101409129ee0000 connected 2024-11-22T16:45:03,159 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:03,160 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:03,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:03,172 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3, hbase.cluster.distributed=false 2024-11-22T16:45:03,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:45:03,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33793 2024-11-22T16:45:03,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33793 2024-11-22T16:45:03,209 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33793 2024-11-22T16:45:03,209 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33793 2024-11-22T16:45:03,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33793 2024-11-22T16:45:03,304 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:45:03,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:03,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:03,307 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:45:03,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:03,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:45:03,312 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T16:45:03,315 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:45:03,316 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38209 2024-11-22T16:45:03,318 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38209 connecting to ZooKeeper ensemble=127.0.0.1:62922 2024-11-22T16:45:03,318 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:03,319 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:03,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382090x0, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:45:03,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38209-0x101409129ee0001 connected 2024-11-22T16:45:03,325 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:03,331 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T16:45:03,344 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.cache 2024-11-22T16:45:03,350 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T16:45:03,351 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T16:45:03,353 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T16:45:03,357 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T16:45:03,357 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T16:45:03,364 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T16:45:03,369 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T16:45:03,369 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T16:45:03,370 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.persistence is complete. 2024-11-22T16:45:03,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T16:45:03,378 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:45:03,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38209 2024-11-22T16:45:03,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38209 2024-11-22T16:45:03,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38209 2024-11-22T16:45:03,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38209 2024-11-22T16:45:03,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38209 2024-11-22T16:45:03,402 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:45:03,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:03,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:03,403 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:45:03,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:03,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:45:03,404 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T16:45:03,405 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:45:03,406 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44213 2024-11-22T16:45:03,408 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44213 connecting to ZooKeeper ensemble=127.0.0.1:62922 2024-11-22T16:45:03,409 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:03,410 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:03,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442130x0, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:45:03,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442130x0, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:03,417 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44213-0x101409129ee0002 connected 2024-11-22T16:45:03,418 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T16:45:03,419 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.cache 2024-11-22T16:45:03,419 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T16:45:03,419 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T16:45:03,419 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T16:45:03,421 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T16:45:03,421 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T16:45:03,421 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T16:45:03,421 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.persistence is complete. 2024-11-22T16:45:03,423 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T16:45:03,424 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T16:45:03,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T16:45:03,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:45:03,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44213 2024-11-22T16:45:03,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44213 2024-11-22T16:45:03,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44213 2024-11-22T16:45:03,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44213 2024-11-22T16:45:03,438 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44213 2024-11-22T16:45:03,459 DEBUG [M:0;4618c6346924:33793 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4618c6346924:33793 2024-11-22T16:45:03,461 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4618c6346924,33793,1732293902475 2024-11-22T16:45:03,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:03,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:03,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:03,471 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4618c6346924,33793,1732293902475 2024-11-22T16:45:03,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T16:45:03,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T16:45:03,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:03,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:03,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:03,497 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T16:45:03,498 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4618c6346924,33793,1732293902475 from backup master directory 2024-11-22T16:45:03,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:03,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:03,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4618c6346924,33793,1732293902475 2024-11-22T16:45:03,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:03,503 WARN [master/4618c6346924:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:45:03,503 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4618c6346924,33793,1732293902475 2024-11-22T16:45:03,505 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T16:45:03,507 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T16:45:03,515 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/hbase.id] with ID: 71a7876b-8b72-408b-aa26-ced197768c10 2024-11-22T16:45:03,516 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.tmp/hbase.id 2024-11-22T16:45:03,522 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/hbase.id] 2024-11-22T16:45:03,563 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:03,569 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T16:45:03,589 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-22T16:45:03,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:03,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:03,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:03,622 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:45:03,625 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T16:45:03,673 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:45:03,680 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:03,741 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store 2024-11-22T16:45:03,765 INFO [master/4618c6346924:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T16:45:03,769 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:03,771 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:45:03,771 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:03,772 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:03,774 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:45:03,774 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:03,774 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:03,775 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732293903771Disabling compacts and flushes for region at 1732293903771Disabling writes for close at 1732293903774 (+3 ms)Writing region close event to WAL at 1732293903774Closed at 1732293903774 2024-11-22T16:45:03,776 WARN [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/.initializing 2024-11-22T16:45:03,776 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/WALs/4618c6346924,33793,1732293902475 2024-11-22T16:45:03,791 INFO [master/4618c6346924:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:45:03,804 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C33793%2C1732293902475, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/WALs/4618c6346924,33793,1732293902475, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/oldWALs, maxLogs=10 2024-11-22T16:45:03,845 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/WALs/4618c6346924,33793,1732293902475/4618c6346924%2C33793%2C1732293902475.1732293903809 2024-11-22T16:45:03,846 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:03,847 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:03,848 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:03,852 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,854 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,896 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T16:45:03,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:03,936 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:03,937 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T16:45:03,943 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:03,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:03,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T16:45:03,951 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:03,951 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:03,952 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T16:45:03,958 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:03,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:03,959 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,963 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,964 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,968 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,969 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,973 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:45:03,976 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:03,985 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:45:03,987 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67200026, jitterRate=0.00135841965675354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:45:03,996 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732293903868Initializing all the Stores at 1732293903869 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293903870 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293903871 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293903871Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293903872 (+1 ms)Cleaning up temporary data from old regions at 1732293903969 (+97 ms)Region opened successfully at 1732293903996 (+27 ms) 2024-11-22T16:45:03,998 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T16:45:04,033 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226a2b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:45:04,069 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T16:45:04,081 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T16:45:04,081 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T16:45:04,085 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:45:04,085 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T16:45:04,091 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:45:04,092 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T16:45:04,125 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T16:45:04,134 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T16:45:04,137 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T16:45:04,139 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T16:45:04,141 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T16:45:04,144 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T16:45:04,147 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T16:45:04,151 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T16:45:04,152 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T16:45:04,154 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T16:45:04,156 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T16:45:04,175 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T16:45:04,177 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T16:45:04,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:04,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:04,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:04,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,195 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4618c6346924,33793,1732293902475, sessionid=0x101409129ee0000, setting cluster-up flag (Was=false) 2024-11-22T16:45:04,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,217 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T16:45:04,219 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,33793,1732293902475 2024-11-22T16:45:04,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,231 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T16:45:04,233 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,33793,1732293902475 2024-11-22T16:45:04,238 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T16:45:04,242 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(746): ClusterId : 71a7876b-8b72-408b-aa26-ced197768c10 2024-11-22T16:45:04,242 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(746): ClusterId : 71a7876b-8b72-408b-aa26-ced197768c10 2024-11-22T16:45:04,245 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T16:45:04,245 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T16:45:04,253 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T16:45:04,253 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T16:45:04,253 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T16:45:04,253 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T16:45:04,257 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T16:45:04,257 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T16:45:04,258 DEBUG [RS:1;4618c6346924:44213 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5573e249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:45:04,258 DEBUG [RS:0;4618c6346924:38209 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368f9d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:45:04,277 DEBUG [RS:0;4618c6346924:38209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4618c6346924:38209 2024-11-22T16:45:04,277 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4618c6346924:44213 2024-11-22T16:45:04,280 INFO [RS:0;4618c6346924:38209 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T16:45:04,280 INFO [RS:1;4618c6346924:44213 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T16:45:04,280 INFO [RS:1;4618c6346924:44213 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T16:45:04,280 INFO [RS:0;4618c6346924:38209 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T16:45:04,280 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T16:45:04,280 DEBUG [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T16:45:04,283 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,33793,1732293902475 with port=38209, startcode=1732293903263 2024-11-22T16:45:04,283 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,33793,1732293902475 with port=44213, startcode=1732293903402 2024-11-22T16:45:04,297 DEBUG [RS:0;4618c6346924:38209 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T16:45:04,297 DEBUG [RS:1;4618c6346924:44213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T16:45:04,331 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:04,344 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T16:45:04,354 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T16:45:04,362 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4618c6346924,33793,1732293902475 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T16:45:04,373 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:04,373 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:04,373 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:04,374 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:04,374 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4618c6346924:0, corePoolSize=10, maxPoolSize=10 2024-11-22T16:45:04,374 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,375 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:45:04,375 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,388 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293934388 2024-11-22T16:45:04,389 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T16:45:04,389 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50775, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T16:45:04,391 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:04,391 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T16:45:04,391 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T16:45:04,392 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T16:45:04,397 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:04,398 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T16:45:04,398 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33793 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:45:04,398 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T16:45:04,398 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T16:45:04,399 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T16:45:04,399 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T16:45:04,406 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33793 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:45:04,404 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,409 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T16:45:04,410 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T16:45:04,410 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T16:45:04,416 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T16:45:04,417 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T16:45:04,419 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293904418,5,FailOnTimeoutGroup] 2024-11-22T16:45:04,428 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293904420,5,FailOnTimeoutGroup] 2024-11-22T16:45:04,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T16:45:04,430 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,430 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,439 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T16:45:04,439 DEBUG [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T16:45:04,439 WARN [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T16:45:04,439 WARN [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T16:45:04,446 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T16:45:04,447 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:04,460 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:04,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T16:45:04,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:45:04,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:04,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:04,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T16:45:04,473 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:45:04,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:04,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:04,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T16:45:04,478 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:45:04,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:04,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:04,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T16:45:04,482 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:45:04,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:04,483 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:04,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:45:04,484 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:04,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:04,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:45:04,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:45:04,486 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:45:04,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:45:04,495 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:45:04,495 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66078425, jitterRate=-0.015354737639427185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:45:04,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732293904460Initializing all the Stores at 1732293904460Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293904461 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293904463 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293904463Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293904463Cleaning up temporary data from old regions at 1732293904486 (+23 ms)Region opened successfully at 1732293904498 (+12 ms) 2024-11-22T16:45:04,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:45:04,499 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:45:04,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:45:04,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:45:04,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:45:04,500 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:45:04,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732293904498Disabling compacts and flushes for region at 1732293904498Disabling writes for close at 1732293904499 (+1 ms)Writing region close event to WAL at 1732293904500 (+1 ms)Closed at 1732293904500 2024-11-22T16:45:04,503 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:04,503 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T16:45:04,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T16:45:04,519 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:45:04,521 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T16:45:04,541 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,33793,1732293902475 with port=44213, startcode=1732293903402 2024-11-22T16:45:04,541 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,33793,1732293902475 with port=38209, startcode=1732293903263 2024-11-22T16:45:04,543 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33793 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4618c6346924,44213,1732293903402 2024-11-22T16:45:04,546 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33793 {}] master.ServerManager(517): Registering regionserver=4618c6346924,44213,1732293903402 2024-11-22T16:45:04,554 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33793 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4618c6346924,38209,1732293903263 2024-11-22T16:45:04,554 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33793 {}] master.ServerManager(517): Registering regionserver=4618c6346924,38209,1732293903263 2024-11-22T16:45:04,554 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:04,554 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T16:45:04,554 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T16:45:04,556 DEBUG [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:04,556 DEBUG [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T16:45:04,557 DEBUG [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T16:45:04,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:04,566 DEBUG [RS:0;4618c6346924:38209 {}] zookeeper.ZKUtil(111): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4618c6346924,38209,1732293903263 2024-11-22T16:45:04,566 DEBUG [RS:1;4618c6346924:44213 {}] zookeeper.ZKUtil(111): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4618c6346924,44213,1732293903402 2024-11-22T16:45:04,566 WARN [RS:1;4618c6346924:44213 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:45:04,566 WARN [RS:0;4618c6346924:38209 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:45:04,566 INFO [RS:1;4618c6346924:44213 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:04,566 INFO [RS:0;4618c6346924:38209 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:04,567 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,44213,1732293903402 2024-11-22T16:45:04,567 DEBUG [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,38209,1732293903263 2024-11-22T16:45:04,568 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4618c6346924,38209,1732293903263] 2024-11-22T16:45:04,568 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4618c6346924,44213,1732293903402] 2024-11-22T16:45:04,595 INFO [RS:1;4618c6346924:44213 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T16:45:04,595 INFO [RS:0;4618c6346924:38209 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T16:45:04,609 INFO [RS:0;4618c6346924:38209 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T16:45:04,609 INFO [RS:1;4618c6346924:44213 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T16:45:04,615 INFO [RS:0;4618c6346924:38209 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T16:45:04,615 INFO [RS:1;4618c6346924:44213 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T16:45:04,616 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,616 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,620 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T16:45:04,620 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T16:45:04,626 INFO [RS:0;4618c6346924:38209 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T16:45:04,626 INFO [RS:1;4618c6346924:44213 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T16:45:04,628 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,628 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,628 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,628 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,629 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:45:04,629 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:45:04,630 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,630 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,631 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,631 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,631 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,631 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:04,631 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:04,631 DEBUG [RS:1;4618c6346924:44213 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:04,631 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:04,631 DEBUG [RS:0;4618c6346924:38209 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:04,634 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,635 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,635 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,635 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,635 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,635 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,44213,1732293903402-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:45:04,636 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,637 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,637 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,637 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,637 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,637 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38209,1732293903263-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:45:04,658 INFO [RS:1;4618c6346924:44213 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T16:45:04,660 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,44213,1732293903402-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,660 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,660 INFO [RS:1;4618c6346924:44213 {}] regionserver.Replication(171): 4618c6346924,44213,1732293903402 started 2024-11-22T16:45:04,664 INFO [RS:0;4618c6346924:38209 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T16:45:04,664 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38209,1732293903263-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,665 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,665 INFO [RS:0;4618c6346924:38209 {}] regionserver.Replication(171): 4618c6346924,38209,1732293903263 started 2024-11-22T16:45:04,672 WARN [4618c6346924:33793 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:45:04,680 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,680 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1482): Serving as 4618c6346924,44213,1732293903402, RpcServer on 4618c6346924/172.17.0.2:44213, sessionid=0x101409129ee0002 2024-11-22T16:45:04,681 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T16:45:04,681 DEBUG [RS:1;4618c6346924:44213 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4618c6346924,44213,1732293903402 2024-11-22T16:45:04,682 DEBUG [RS:1;4618c6346924:44213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,44213,1732293903402' 2024-11-22T16:45:04,682 DEBUG [RS:1;4618c6346924:44213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T16:45:04,683 DEBUG [RS:1;4618c6346924:44213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T16:45:04,684 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T16:45:04,684 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T16:45:04,684 DEBUG [RS:1;4618c6346924:44213 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4618c6346924,44213,1732293903402 2024-11-22T16:45:04,684 DEBUG [RS:1;4618c6346924:44213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,44213,1732293903402' 2024-11-22T16:45:04,684 DEBUG [RS:1;4618c6346924:44213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T16:45:04,685 DEBUG [RS:1;4618c6346924:44213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T16:45:04,685 DEBUG [RS:1;4618c6346924:44213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T16:45:04,686 INFO [RS:1;4618c6346924:44213 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T16:45:04,686 INFO [RS:1;4618c6346924:44213 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T16:45:04,689 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:04,689 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(1482): Serving as 4618c6346924,38209,1732293903263, RpcServer on 4618c6346924/172.17.0.2:38209, sessionid=0x101409129ee0001 2024-11-22T16:45:04,690 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T16:45:04,690 DEBUG [RS:0;4618c6346924:38209 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4618c6346924,38209,1732293903263 2024-11-22T16:45:04,690 DEBUG [RS:0;4618c6346924:38209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,38209,1732293903263' 2024-11-22T16:45:04,690 DEBUG [RS:0;4618c6346924:38209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T16:45:04,691 DEBUG [RS:0;4618c6346924:38209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T16:45:04,691 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T16:45:04,691 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T16:45:04,691 DEBUG [RS:0;4618c6346924:38209 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4618c6346924,38209,1732293903263 2024-11-22T16:45:04,692 DEBUG [RS:0;4618c6346924:38209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,38209,1732293903263' 2024-11-22T16:45:04,692 DEBUG [RS:0;4618c6346924:38209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T16:45:04,692 DEBUG [RS:0;4618c6346924:38209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T16:45:04,693 DEBUG [RS:0;4618c6346924:38209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T16:45:04,693 INFO [RS:0;4618c6346924:38209 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T16:45:04,693 INFO [RS:0;4618c6346924:38209 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T16:45:04,791 INFO [RS:1;4618c6346924:44213 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:45:04,792 INFO [RS:1;4618c6346924:44213 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4618c6346924%2C44213%2C1732293903402, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,44213,1732293903402, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs, maxLogs=55 2024-11-22T16:45:04,794 INFO [RS:0;4618c6346924:38209 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:45:04,795 INFO [RS:0;4618c6346924:38209 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4618c6346924%2C38209%2C1732293903263, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,38209,1732293903263, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs, maxLogs=55 2024-11-22T16:45:04,813 INFO [RS:1;4618c6346924:44213 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,44213,1732293903402/4618c6346924%2C44213%2C1732293903402.1732293904794 2024-11-22T16:45:04,814 DEBUG [RS:1;4618c6346924:44213 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:04,819 INFO [RS:0;4618c6346924:38209 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,38209,1732293903263/4618c6346924%2C38209%2C1732293903263.1732293904798 2024-11-22T16:45:04,819 DEBUG [RS:0;4618c6346924:38209 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:04,925 DEBUG [4618c6346924:33793 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T16:45:04,937 DEBUG [4618c6346924:33793 {}] balancer.BalancerClusterState(204): Hosts are {4618c6346924=0} racks are {/default-rack=0} 2024-11-22T16:45:04,940 DEBUG [4618c6346924:33793 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T16:45:04,940 DEBUG [4618c6346924:33793 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T16:45:04,940 DEBUG [4618c6346924:33793 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T16:45:04,940 DEBUG [4618c6346924:33793 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T16:45:04,940 INFO [4618c6346924:33793 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T16:45:04,940 INFO [4618c6346924:33793 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T16:45:04,940 DEBUG [4618c6346924:33793 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T16:45:04,947 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4618c6346924,44213,1732293903402 2024-11-22T16:45:04,953 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,44213,1732293903402, state=OPENING 2024-11-22T16:45:04,958 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T16:45:04,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:04,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:04,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:04,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:04,962 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:45:04,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,44213,1732293903402}] 2024-11-22T16:45:05,137 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T16:45:05,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52737, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T16:45:05,150 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T16:45:05,151 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:05,151 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T16:45:05,152 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4618c6346924%2C44213%2C1732293903402.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,44213,1732293903402, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs, maxLogs=55 2024-11-22T16:45:05,173 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,44213,1732293903402/4618c6346924%2C44213%2C1732293903402.meta.1732293905154.meta 2024-11-22T16:45:05,174 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:05,175 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:05,177 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T16:45:05,181 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T16:45:05,186 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T16:45:05,192 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T16:45:05,193 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:05,193 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T16:45:05,194 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T16:45:05,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T16:45:05,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:45:05,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:05,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:05,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T16:45:05,198 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:45:05,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:05,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:05,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T16:45:05,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:45:05,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:05,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:05,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T16:45:05,200 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:45:05,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:05,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:05,201 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:45:05,202 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:05,203 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:05,203 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:45:05,203 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:45:05,204 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:45:05,205 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:45:05,205 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62098101, jitterRate=-0.07466618716716766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:45:05,206 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T16:45:05,207 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732293905194Writing region info on filesystem at 1732293905195 (+1 ms)Initializing all the Stores at 1732293905195Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293905195Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293905195Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293905196 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293905196Cleaning up temporary data from old regions at 1732293905203 (+7 ms)Running coprocessor post-open hooks at 1732293905206 (+3 ms)Region opened successfully at 1732293905207 (+1 ms) 2024-11-22T16:45:05,213 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732293905128 2024-11-22T16:45:05,224 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T16:45:05,225 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T16:45:05,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,44213,1732293903402 2024-11-22T16:45:05,227 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,44213,1732293903402, state=OPEN 2024-11-22T16:45:05,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:45:05,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:45:05,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:45:05,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:05,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:05,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:05,230 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4618c6346924,44213,1732293903402 2024-11-22T16:45:05,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T16:45:05,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,44213,1732293903402 in 266 msec 2024-11-22T16:45:05,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T16:45:05,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 725 msec 2024-11-22T16:45:05,238 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:05,238 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T16:45:05,261 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:45:05,262 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,44213,1732293903402, seqNum=-1] 2024-11-22T16:45:05,281 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:45:05,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51935, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:45:05,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0260 sec 2024-11-22T16:45:05,302 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732293905302, completionTime=-1 2024-11-22T16:45:05,304 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T16:45:05,304 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:45:05,330 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T16:45:05,330 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293965330 2024-11-22T16:45:05,331 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732294025331 2024-11-22T16:45:05,331 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-22T16:45:05,333 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-22T16:45:05,339 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33793,1732293902475-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:05,339 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33793,1732293902475-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:05,339 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33793,1732293902475-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:05,341 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4618c6346924:33793, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:05,341 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:05,342 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:05,349 DEBUG [master/4618c6346924:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T16:45:05,370 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.867sec 2024-11-22T16:45:05,371 INFO [master/4618c6346924:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T16:45:05,373 INFO [master/4618c6346924:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T16:45:05,374 INFO [master/4618c6346924:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T16:45:05,374 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T16:45:05,374 INFO [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T16:45:05,375 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33793,1732293902475-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:45:05,375 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33793,1732293902475-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T16:45:05,376 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T16:45:05,377 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T16:45:05,377 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33793,1732293902475-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:05,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:45:05,455 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T16:45:05,455 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T16:45:05,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4618c6346924,33793,-1 for getting cluster id 2024-11-22T16:45:05,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T16:45:05,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '71a7876b-8b72-408b-aa26-ced197768c10' 2024-11-22T16:45:05,474 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T16:45:05,474 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "71a7876b-8b72-408b-aa26-ced197768c10" 2024-11-22T16:45:05,477 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc648fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:45:05,477 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4618c6346924,33793,-1] 2024-11-22T16:45:05,480 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T16:45:05,482 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:05,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T16:45:05,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f82a59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:45:05,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:45:05,493 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,44213,1732293903402, seqNum=-1] 2024-11-22T16:45:05,493 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:45:05,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:45:05,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4618c6346924,33793,1732293902475 2024-11-22T16:45:05,531 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T16:45:05,535 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4618c6346924,33793,1732293902475 2024-11-22T16:45:05,538 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4234e5fb 2024-11-22T16:45:05,541 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T16:45:05,545 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T16:45:05,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732293905526', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:45:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 2024-11-22T16:45:05,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:45:05,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732293905526" procId is: 4 2024-11-22T16:45:05,561 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:05,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:45:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:05,576 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a220efbdf14ceeef3fcfb704e47703bc, NAME => 'testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732293905526', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:05,583 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:05,583 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(1722): Closing a220efbdf14ceeef3fcfb704e47703bc, disabling compactions & flushes 2024-11-22T16:45:05,583 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:05,583 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:05,583 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. after waiting 0 ms 2024-11-22T16:45:05,583 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:05,583 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:05,584 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732293905526-pool-0 {}] regionserver.HRegion(1676): Region close journal for a220efbdf14ceeef3fcfb704e47703bc: Waiting for close lock at 1732293905583Disabling compacts and flushes for region at 1732293905583Disabling writes for close at 1732293905583Writing region close event to WAL at 1732293905583Closed at 1732293905583 2024-11-22T16:45:05,585 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:45:05,590 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732293905585"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293905585"}]},"ts":"1732293905585"} 2024-11-22T16:45:05,594 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:45:05,595 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:45:05,598 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732293905526","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293905595"}]},"ts":"1732293905595"} 2024-11-22T16:45:05,602 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732293905526, state=ENABLING in hbase:meta 2024-11-22T16:45:05,602 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4618c6346924=0} racks are {/default-rack=0} 2024-11-22T16:45:05,603 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T16:45:05,603 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T16:45:05,603 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T16:45:05,603 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T16:45:05,603 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T16:45:05,603 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T16:45:05,603 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T16:45:05,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, ASSIGN}] 2024-11-22T16:45:05,606 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, ASSIGN 2024-11-22T16:45:05,607 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, ASSIGN; state=OFFLINE, location=4618c6346924,38209,1732293903263; forceNewPlan=false, retain=false 2024-11-22T16:45:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:05,759 INFO [4618c6346924:33793 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T16:45:05,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a220efbdf14ceeef3fcfb704e47703bc, regionState=OPENING, regionLocation=4618c6346924,38209,1732293903263 2024-11-22T16:45:05,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, ASSIGN because future has completed 2024-11-22T16:45:05,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,38209,1732293903263}] 2024-11-22T16:45:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:05,918 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T16:45:05,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58911, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T16:45:05,923 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:05,924 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a220efbdf14ceeef3fcfb704e47703bc, NAME => 'testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:05,924 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732293905526 a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,925 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:05,925 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,925 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,926 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,931 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a220efbdf14ceeef3fcfb704e47703bc columnFamilyName family 2024-11-22T16:45:05,931 DEBUG [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:05,932 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.HStore(327): Store=a220efbdf14ceeef3fcfb704e47703bc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:05,932 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,933 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,933 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,933 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,933 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,934 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,941 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:45:05,941 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a220efbdf14ceeef3fcfb704e47703bc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74658622, jitterRate=0.1125001609325409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:45:05,941 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:05,942 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a220efbdf14ceeef3fcfb704e47703bc: Running coprocessor pre-open hook at 1732293905925Writing region info on filesystem at 1732293905925Initializing all the Stores at 1732293905926 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293905926Cleaning up temporary data from old regions at 1732293905934 (+8 ms)Running coprocessor post-open hooks at 1732293905941 (+7 ms)Region opened successfully at 1732293905942 (+1 ms) 2024-11-22T16:45:05,943 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc., pid=6, masterSystemTime=1732293905918 2024-11-22T16:45:05,946 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:05,946 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:05,947 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a220efbdf14ceeef3fcfb704e47703bc, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,38209,1732293903263 2024-11-22T16:45:05,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,38209,1732293903263 because future has completed 2024-11-22T16:45:05,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T16:45:05,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,38209,1732293903263 in 186 msec 2024-11-22T16:45:05,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T16:45:05,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, ASSIGN in 349 msec 2024-11-22T16:45:05,956 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:45:05,956 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732293905526","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293905956"}]},"ts":"1732293905956"} 2024-11-22T16:45:05,958 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732293905526, state=ENABLED in hbase:meta 2024-11-22T16:45:05,959 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:45:05,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732293905526 in 406 msec 2024-11-22T16:45:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:06,201 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732293905526 completed 2024-11-22T16:45:06,201 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732293905526 get assigned. Timeout = 60000ms 2024-11-22T16:45:06,204 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:45:06,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732293905526 assigned to meta. Checking AM states. 2024-11-22T16:45:06,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:45:06,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732293905526 assigned. 2024-11-22T16:45:06,218 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732293905526', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc., hostname=4618c6346924,38209,1732293903263, seqNum=2] 2024-11-22T16:45:06,220 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:45:06,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:45:06,232 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a220efbdf14ceeef3fcfb704e47703bc 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T16:45:06,258 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:06,300 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/.tmp/family/9ed3e97eb79e4a7aa09e341e58a745cf is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T16:45:06,305 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/.tmp/family/9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:06,342 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/.tmp/family/9ed3e97eb79e4a7aa09e341e58a745cf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:06,344 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf, delay=946 ms 2024-11-22T16:45:06,345 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T16:45:06,352 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for a220efbdf14ceeef3fcfb704e47703bc in 119ms, sequenceid=6, compaction requested=false 2024-11-22T16:45:06,353 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732293905526' 2024-11-22T16:45:06,354 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a220efbdf14ceeef3fcfb704e47703bc: 2024-11-22T16:45:07,292 DEBUG [hfile-prefetch-1732293906344 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9ed3e97eb79e4a7aa09e341e58a745cf_0. 2024-11-22T16:45:07,296 DEBUG [hfile-prefetch-1732293906344 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9ed3e97eb79e4a7aa09e341e58a745cf_117. 2024-11-22T16:45:07,297 DEBUG [hfile-prefetch-1732293906344 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf, with total blocks 2, and data blocks 1 2024-11-22T16:45:07,297 DEBUG [hfile-prefetch-1732293906344 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T16:45:07,298 DEBUG [hfile-prefetch-1732293906344 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T16:45:07,298 DEBUG [hfile-prefetch-1732293906344 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf. Will sleep 100ms and try the verification again. 2024-11-22T16:45:07,323 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T16:45:07,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T16:45:07,324 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:07,324 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:07,339 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T16:45:07,339 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732293907323 2024-11-22T16:45:07,339 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T16:45:07,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=a220efbdf14ceeef3fcfb704e47703bc, source=4618c6346924,38209,1732293903263, destination=4618c6346924,44213,1732293903402, warming up region on 4618c6346924,44213,1732293903402 2024-11-22T16:45:07,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=a220efbdf14ceeef3fcfb704e47703bc, source=4618c6346924,38209,1732293903263, destination=4618c6346924,44213,1732293903402, running balancer 2024-11-22T16:45:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, REOPEN/MOVE 2024-11-22T16:45:07,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,368 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, REOPEN/MOVE 2024-11-22T16:45:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(7855): Warmup {ENCODED => a220efbdf14ceeef3fcfb704e47703bc, NAME => 'testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:07,369 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,370 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a220efbdf14ceeef3fcfb704e47703bc, regionState=CLOSING, regionLocation=4618c6346924,38209,1732293903263 2024-11-22T16:45:07,371 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a220efbdf14ceeef3fcfb704e47703bc columnFamilyName family 2024-11-22T16:45:07,371 DEBUG [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:07,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, REOPEN/MOVE because future has completed 2024-11-22T16:45:07,374 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-22T16:45:07,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,38209,1732293903263}] 2024-11-22T16:45:07,375 DEBUG [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:07,375 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.HStore(327): Store=a220efbdf14ceeef3fcfb704e47703bc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(1722): Closing a220efbdf14ceeef3fcfb704e47703bc, disabling compactions & flushes 2024-11-22T16:45:07,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. after waiting 0 ms 2024-11-22T16:45:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,377 DEBUG [StoreFileCloser-a220efbdf14ceeef3fcfb704e47703bc-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:07,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44213 {}] regionserver.HRegion(1676): Region close journal for a220efbdf14ceeef3fcfb704e47703bc: Waiting for close lock at 1732293907375Disabling compacts and flushes for region at 1732293907375Disabling writes for close at 1732293907375Writing region close event to WAL at 1732293907377 (+2 ms)Closed at 1732293907377 2024-11-22T16:45:07,378 WARN [hfile-prefetch-1732293906344 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T16:45:07,379 DEBUG [hfile-prefetch-1732293906344 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33793 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-22T16:45:07,532 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,533 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-22T16:45:07,533 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing a220efbdf14ceeef3fcfb704e47703bc, disabling compactions & flushes 2024-11-22T16:45:07,534 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,534 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,534 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. after waiting 0 ms 2024-11-22T16:45:07,534 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,535 DEBUG [StoreFileCloser-a220efbdf14ceeef3fcfb704e47703bc-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 9ed3e97eb79e4a7aa09e341e58a745cf, starting offset: 0, end offset: 9223372036854775807 2024-11-22T16:45:07,537 DEBUG [StoreFileCloser-a220efbdf14ceeef3fcfb704e47703bc-family-1 {}] bucket.BucketCache(856): removed key 9ed3e97eb79e4a7aa09e341e58a745cf_0 from back map with offset lock 2094080 in the evict process 2024-11-22T16:45:07,539 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T16:45:07,539 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T16:45:07,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:07,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:07,542 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T16:45:07,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T16:45:07,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732293907539 2024-11-22T16:45:07,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T16:45:07,543 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,544 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for a220efbdf14ceeef3fcfb704e47703bc: Waiting for close lock at 1732293907533Running coprocessor pre-close hooks at 1732293907533Disabling compacts and flushes for region at 1732293907533Disabling writes for close at 1732293907534 (+1 ms)Writing region close event to WAL at 1732293907537 (+3 ms)Running coprocessor post-close hooks at 1732293907542 (+5 ms)Closed at 1732293907543 (+1 ms) 2024-11-22T16:45:07,544 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding a220efbdf14ceeef3fcfb704e47703bc move to 4618c6346924,44213,1732293903402 record at close sequenceid=6 2024-11-22T16:45:07,547 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a220efbdf14ceeef3fcfb704e47703bc, regionState=CLOSED 2024-11-22T16:45:07,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,38209,1732293903263 because future has completed 2024-11-22T16:45:07,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T16:45:07,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,38209,1732293903263 in 177 msec 2024-11-22T16:45:07,554 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, REOPEN/MOVE; state=CLOSED, location=4618c6346924,44213,1732293903402; forceNewPlan=false, retain=false 2024-11-22T16:45:07,705 INFO [4618c6346924:33793 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T16:45:07,705 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a220efbdf14ceeef3fcfb704e47703bc, regionState=OPENING, regionLocation=4618c6346924,44213,1732293903402 2024-11-22T16:45:07,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, REOPEN/MOVE because future has completed 2024-11-22T16:45:07,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,44213,1732293903402}] 2024-11-22T16:45:07,864 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,865 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => a220efbdf14ceeef3fcfb704e47703bc, NAME => 'testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:07,865 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732293905526 a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,865 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:07,866 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,866 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,867 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,868 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a220efbdf14ceeef3fcfb704e47703bc columnFamilyName family 2024-11-22T16:45:07,868 DEBUG [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:07,870 DEBUG [StoreFileOpener-a220efbdf14ceeef3fcfb704e47703bc-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf, delay=936 ms 2024-11-22T16:45:07,872 DEBUG [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:07,872 INFO [StoreOpener-a220efbdf14ceeef3fcfb704e47703bc-1 {}] regionserver.HStore(327): Store=a220efbdf14ceeef3fcfb704e47703bc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:07,872 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,873 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,873 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,874 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,874 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,878 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,879 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened a220efbdf14ceeef3fcfb704e47703bc; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69175118, jitterRate=0.030789583921432495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:45:07,879 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,880 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for a220efbdf14ceeef3fcfb704e47703bc: Running coprocessor pre-open hook at 1732293907866Writing region info on filesystem at 1732293907866Initializing all the Stores at 1732293907866Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293907866Cleaning up temporary data from old regions at 1732293907874 (+8 ms)Running coprocessor post-open hooks at 1732293907879 (+5 ms)Region opened successfully at 1732293907880 (+1 ms) 2024-11-22T16:45:07,880 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc., pid=9, masterSystemTime=1732293907862 2024-11-22T16:45:07,883 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,883 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,884 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a220efbdf14ceeef3fcfb704e47703bc, regionState=OPEN, openSeqNum=10, regionLocation=4618c6346924,44213,1732293903402 2024-11-22T16:45:07,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,44213,1732293903402 because future has completed 2024-11-22T16:45:07,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-22T16:45:07,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure a220efbdf14ceeef3fcfb704e47703bc, server=4618c6346924,44213,1732293903402 in 179 msec 2024-11-22T16:45:07,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732293905526, region=a220efbdf14ceeef3fcfb704e47703bc, REOPEN/MOVE in 524 msec 2024-11-22T16:45:07,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T16:45:07,921 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:45:07,921 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:07,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:07,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:07,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T16:45:07,926 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T16:45:07,927 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-22T16:45:07,927 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4618c6346924,33793,1732293902475 2024-11-22T16:45:07,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:07,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:07,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:07,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:07,929 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:45:07,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:07,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:07,930 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:45:07,930 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:07,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:07,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:07,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:07,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:07,931 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4618c6346924,38209,1732293903263' ***** 2024-11-22T16:45:07,931 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T16:45:07,931 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4618c6346924,44213,1732293903402' ***** 2024-11-22T16:45:07,931 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T16:45:07,931 INFO [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T16:45:07,931 INFO [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T16:45:07,932 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T16:45:07,932 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T16:45:07,932 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T16:45:07,932 INFO [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.persistence 2024-11-22T16:45:07,932 INFO [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/bucket.persistence 2024-11-22T16:45:07,932 INFO [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T16:45:07,932 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T16:45:07,932 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T16:45:07,932 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T16:45:07,932 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T16:45:07,933 INFO [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T16:45:07,933 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T16:45:07,938 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T16:45:07,938 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T16:45:07,939 INFO [regionserver/4618c6346924:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:07,940 INFO [regionserver/4618c6346924:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:07,942 DEBUG [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1487): Thread RS:0;4618c6346924:38209 started persisting bucket cache to file 2024-11-22T16:45:07,942 DEBUG [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:07,942 DEBUG [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:07,943 DEBUG [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1487): Thread RS:1;4618c6346924:44213 started persisting bucket cache to file 2024-11-22T16:45:07,943 DEBUG [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:07,943 DEBUG [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:07,946 DEBUG [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T16:45:07,946 DEBUG [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732293907942 2024-11-22T16:45:07,946 DEBUG [RS:0;4618c6346924:38209 {}] bucket.BucketCache(1509): Thread RS:0;4618c6346924:38209 finished persisting bucket cache to file, renaming 2024-11-22T16:45:07,946 DEBUG [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T16:45:07,946 DEBUG [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732293907943 2024-11-22T16:45:07,946 DEBUG [RS:1;4618c6346924:44213 {}] bucket.BucketCache(1509): Thread RS:1;4618c6346924:44213 finished persisting bucket cache to file, renaming 2024-11-22T16:45:07,947 INFO [RS:0;4618c6346924:38209 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T16:45:07,947 INFO [RS:0;4618c6346924:38209 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T16:45:07,947 INFO [RS:0;4618c6346924:38209 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T16:45:07,947 INFO [RS:1;4618c6346924:44213 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T16:45:07,947 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(959): stopping server 4618c6346924,38209,1732293903263 2024-11-22T16:45:07,947 INFO [RS:1;4618c6346924:44213 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T16:45:07,947 INFO [RS:0;4618c6346924:38209 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:45:07,947 INFO [RS:1;4618c6346924:44213 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T16:45:07,947 INFO [RS:0;4618c6346924:38209 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4618c6346924:38209. 2024-11-22T16:45:07,947 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(3091): Received CLOSE for a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,947 DEBUG [RS:0;4618c6346924:38209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:07,947 DEBUG [RS:0;4618c6346924:38209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:07,948 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(976): stopping server 4618c6346924,38209,1732293903263; all regions closed. 2024-11-22T16:45:07,948 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(959): stopping server 4618c6346924,44213,1732293903402 2024-11-22T16:45:07,948 INFO [RS:1;4618c6346924:44213 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:45:07,948 INFO [RS:1;4618c6346924:44213 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4618c6346924:44213. 2024-11-22T16:45:07,948 DEBUG [RS:1;4618c6346924:44213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:07,948 DEBUG [RS:1;4618c6346924:44213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:07,948 INFO [RS:1;4618c6346924:44213 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T16:45:07,948 INFO [RS:1;4618c6346924:44213 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T16:45:07,948 INFO [RS:1;4618c6346924:44213 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T16:45:07,948 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T16:45:07,948 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a220efbdf14ceeef3fcfb704e47703bc, disabling compactions & flushes 2024-11-22T16:45:07,949 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,949 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. after waiting 0 ms 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,949 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1325): Online Regions={a220efbdf14ceeef3fcfb704e47703bc=testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc., 1588230740=hbase:meta,,1.1588230740} 2024-11-22T16:45:07,949 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:45:07,949 DEBUG [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a220efbdf14ceeef3fcfb704e47703bc 2024-11-22T16:45:07,949 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:45:07,949 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:45:07,950 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-22T16:45:07,950 DEBUG [StoreFileCloser-a220efbdf14ceeef3fcfb704e47703bc-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/family/9ed3e97eb79e4a7aa09e341e58a745cf 2024-11-22T16:45:07,953 DEBUG [RS:0;4618c6346924:38209 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs 2024-11-22T16:45:07,954 INFO [RS:0;4618c6346924:38209 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C38209%2C1732293903263:(num 1732293904798) 2024-11-22T16:45:07,954 DEBUG [RS:0;4618c6346924:38209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:07,954 INFO [RS:0;4618c6346924:38209 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:07,954 INFO [RS:0;4618c6346924:38209 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:45:07,954 INFO [RS:0;4618c6346924:38209 {}] hbase.ChoreService(370): Chore service for: regionserver/4618c6346924:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T16:45:07,955 INFO [RS:0;4618c6346924:38209 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T16:45:07,955 INFO [regionserver/4618c6346924:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:45:07,955 INFO [RS:0;4618c6346924:38209 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T16:45:07,955 INFO [RS:0;4618c6346924:38209 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T16:45:07,955 INFO [RS:0;4618c6346924:38209 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:45:07,955 INFO [RS:0;4618c6346924:38209 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38209 2024-11-22T16:45:07,959 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnRegionMove1732293905526/a220efbdf14ceeef3fcfb704e47703bc/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T16:45:07,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4618c6346924,38209,1732293903263 2024-11-22T16:45:07,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:07,960 INFO [RS:0;4618c6346924:38209 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:45:07,960 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,960 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a220efbdf14ceeef3fcfb704e47703bc: Waiting for close lock at 1732293907948Running coprocessor pre-close hooks at 1732293907949 (+1 ms)Disabling compacts and flushes for region at 1732293907949Disabling writes for close at 1732293907949Writing region close event to WAL at 1732293907951 (+2 ms)Running coprocessor post-close hooks at 1732293907960 (+9 ms)Closed at 1732293907960 2024-11-22T16:45:07,960 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc. 2024-11-22T16:45:07,961 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4618c6346924,38209,1732293903263] 2024-11-22T16:45:07,963 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4618c6346924,38209,1732293903263 already deleted, retry=false 2024-11-22T16:45:07,963 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4618c6346924,38209,1732293903263 expired; onlineServers=1 2024-11-22T16:45:07,965 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8318140850e64a82aca193c5b2e5f4d6 2024-11-22T16:45:07,983 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/info/8318140850e64a82aca193c5b2e5f4d6 is 205, key is testBlockEvictionOnRegionMove1732293905526,,1732293905546.a220efbdf14ceeef3fcfb704e47703bc./info:regioninfo/1732293907884/Put/seqid=0 2024-11-22T16:45:07,984 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/info/8318140850e64a82aca193c5b2e5f4d6 2024-11-22T16:45:07,993 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fd3575e578f14b11848971349b4999fc 2024-11-22T16:45:08,007 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/ns/fd3575e578f14b11848971349b4999fc is 43, key is default/ns:d/1732293905289/Put/seqid=0 2024-11-22T16:45:08,008 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/ns/fd3575e578f14b11848971349b4999fc 2024-11-22T16:45:08,017 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b49209174a2342f8ab38014fa509bf9f 2024-11-22T16:45:08,030 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/table/b49209174a2342f8ab38014fa509bf9f is 78, key is testBlockEvictionOnRegionMove1732293905526/table:state/1732293905956/Put/seqid=0 2024-11-22T16:45:08,031 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/table/b49209174a2342f8ab38014fa509bf9f 2024-11-22T16:45:08,034 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/info/8318140850e64a82aca193c5b2e5f4d6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/8318140850e64a82aca193c5b2e5f4d6 2024-11-22T16:45:08,035 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/8318140850e64a82aca193c5b2e5f4d6, delay=954 ms 2024-11-22T16:45:08,036 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/8318140850e64a82aca193c5b2e5f4d6, entries=15, sequenceid=15, filesize=7.6 K 2024-11-22T16:45:08,037 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/ns/fd3575e578f14b11848971349b4999fc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/fd3575e578f14b11848971349b4999fc 2024-11-22T16:45:08,038 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/fd3575e578f14b11848971349b4999fc, delay=953 ms 2024-11-22T16:45:08,039 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/fd3575e578f14b11848971349b4999fc, entries=2, sequenceid=15, filesize=5.0 K 2024-11-22T16:45:08,040 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/table/b49209174a2342f8ab38014fa509bf9f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/b49209174a2342f8ab38014fa509bf9f 2024-11-22T16:45:08,041 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/b49209174a2342f8ab38014fa509bf9f, delay=946 ms 2024-11-22T16:45:08,042 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/b49209174a2342f8ab38014fa509bf9f, entries=2, sequenceid=15, filesize=5.3 K 2024-11-22T16:45:08,043 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-11-22T16:45:08,043 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T16:45:08,044 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/8318140850e64a82aca193c5b2e5f4d6 2024-11-22T16:45:08,045 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/fd3575e578f14b11848971349b4999fc 2024-11-22T16:45:08,046 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/b49209174a2342f8ab38014fa509bf9f 2024-11-22T16:45:08,050 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-22T16:45:08,051 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T16:45:08,051 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:45:08,052 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732293907949Running coprocessor pre-close hooks at 1732293907949Disabling compacts and flushes for region at 1732293907949Disabling writes for close at 1732293907949Obtaining lock to block concurrent updates at 1732293907950 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732293907950Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732293907950Flushing stores of hbase:meta,,1.1588230740 at 1732293907951 (+1 ms)Flushing 1588230740/info: creating writer at 1732293907951Flushing 1588230740/info: appending metadata at 1732293907982 (+31 ms)Flushing 1588230740/info: closing flushed file at 1732293907982Flushing 1588230740/ns: creating writer at 1732293907987 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732293908006 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732293908006Flushing 1588230740/table: creating writer at 1732293908010 (+4 ms)Flushing 1588230740/table: appending metadata at 1732293908030 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732293908030Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d95f45: reopening flushed file at 1732293908033 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2163d2d1: reopening flushed file at 1732293908036 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@707598f8: reopening flushed file at 1732293908039 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1732293908043 (+4 ms)Writing region close event to WAL at 1732293908046 (+3 ms)Running coprocessor post-close hooks at 1732293908051 (+5 ms)Closed at 1732293908051 2024-11-22T16:45:08,052 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T16:45:08,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:08,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38209-0x101409129ee0001, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:08,063 INFO [RS:0;4618c6346924:38209 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:45:08,063 INFO [RS:0;4618c6346924:38209 {}] regionserver.HRegionServer(1031): Exiting; stopping=4618c6346924,38209,1732293903263; zookeeper connection closed. 2024-11-22T16:45:08,063 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@705f72b1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@705f72b1 2024-11-22T16:45:08,149 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(976): stopping server 4618c6346924,44213,1732293903402; all regions closed. 2024-11-22T16:45:08,152 DEBUG [RS:1;4618c6346924:44213 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs 2024-11-22T16:45:08,152 INFO [RS:1;4618c6346924:44213 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C44213%2C1732293903402.meta:.meta(num 1732293905154) 2024-11-22T16:45:08,153 DEBUG [RS:1;4618c6346924:44213 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs 2024-11-22T16:45:08,154 INFO [RS:1;4618c6346924:44213 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C44213%2C1732293903402:(num 1732293904794) 2024-11-22T16:45:08,154 DEBUG [RS:1;4618c6346924:44213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:08,154 INFO [RS:1;4618c6346924:44213 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:08,154 INFO [RS:1;4618c6346924:44213 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:45:08,154 INFO [RS:1;4618c6346924:44213 {}] hbase.ChoreService(370): Chore service for: regionserver/4618c6346924:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T16:45:08,154 INFO [RS:1;4618c6346924:44213 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:45:08,154 INFO [regionserver/4618c6346924:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:45:08,154 INFO [RS:1;4618c6346924:44213 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44213 2024-11-22T16:45:08,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4618c6346924,44213,1732293903402 2024-11-22T16:45:08,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:08,157 INFO [RS:1;4618c6346924:44213 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:45:08,159 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4618c6346924,44213,1732293903402] 2024-11-22T16:45:08,161 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4618c6346924,44213,1732293903402 already deleted, retry=false 2024-11-22T16:45:08,161 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4618c6346924,44213,1732293903402 expired; onlineServers=0 2024-11-22T16:45:08,161 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4618c6346924,33793,1732293902475' ***** 2024-11-22T16:45:08,161 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T16:45:08,161 INFO [M:0;4618c6346924:33793 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:45:08,161 INFO [M:0;4618c6346924:33793 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:45:08,161 DEBUG [M:0;4618c6346924:33793 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T16:45:08,162 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T16:45:08,162 DEBUG [M:0;4618c6346924:33793 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T16:45:08,162 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293904420 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293904420,5,FailOnTimeoutGroup] 2024-11-22T16:45:08,162 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293904418 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293904418,5,FailOnTimeoutGroup] 2024-11-22T16:45:08,162 INFO [M:0;4618c6346924:33793 {}] hbase.ChoreService(370): Chore service for: master/4618c6346924:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T16:45:08,162 INFO [M:0;4618c6346924:33793 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:45:08,162 DEBUG [M:0;4618c6346924:33793 {}] master.HMaster(1795): Stopping service threads 2024-11-22T16:45:08,162 INFO [M:0;4618c6346924:33793 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T16:45:08,162 INFO [M:0;4618c6346924:33793 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:45:08,163 INFO [M:0;4618c6346924:33793 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:45:08,163 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T16:45:08,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T16:45:08,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,164 DEBUG [M:0;4618c6346924:33793 {}] zookeeper.ZKUtil(347): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T16:45:08,164 WARN [M:0;4618c6346924:33793 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T16:45:08,164 INFO [M:0;4618c6346924:33793 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.lastflushedseqids 2024-11-22T16:45:08,177 INFO [M:0;4618c6346924:33793 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:45:08,177 INFO [M:0;4618c6346924:33793 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T16:45:08,177 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:45:08,178 INFO [M:0;4618c6346924:33793 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,178 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,178 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:45:08,178 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,178 INFO [M:0;4618c6346924:33793 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-22T16:45:08,188 DEBUG [M:0;4618c6346924:33793 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c568f86722ac4bd189c0ecd869aa006e 2024-11-22T16:45:08,202 DEBUG [M:0;4618c6346924:33793 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c568f86722ac4bd189c0ecd869aa006e is 82, key is hbase:meta,,1/info:regioninfo/1732293905225/Put/seqid=0 2024-11-22T16:45:08,203 INFO [M:0;4618c6346924:33793 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c568f86722ac4bd189c0ecd869aa006e 2024-11-22T16:45:08,212 DEBUG [M:0;4618c6346924:33793 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4f1f9fe63deb4888a5fed246b433b242 2024-11-22T16:45:08,227 DEBUG [M:0;4618c6346924:33793 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f1f9fe63deb4888a5fed246b433b242 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732293905960/Put/seqid=0 2024-11-22T16:45:08,229 INFO [M:0;4618c6346924:33793 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f1f9fe63deb4888a5fed246b433b242 2024-11-22T16:45:08,238 DEBUG [M:0;4618c6346924:33793 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dadd46505e164e83a398c51e7f3e2d31 2024-11-22T16:45:08,251 DEBUG [M:0;4618c6346924:33793 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dadd46505e164e83a398c51e7f3e2d31 is 69, key is 4618c6346924,38209,1732293903263/rs:state/1732293904554/Put/seqid=0 2024-11-22T16:45:08,252 INFO [M:0;4618c6346924:33793 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dadd46505e164e83a398c51e7f3e2d31 2024-11-22T16:45:08,255 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c568f86722ac4bd189c0ecd869aa006e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c568f86722ac4bd189c0ecd869aa006e 2024-11-22T16:45:08,257 INFO [M:0;4618c6346924:33793 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c568f86722ac4bd189c0ecd869aa006e, entries=8, sequenceid=84, filesize=5.5 K 2024-11-22T16:45:08,258 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f1f9fe63deb4888a5fed246b433b242 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f1f9fe63deb4888a5fed246b433b242 2024-11-22T16:45:08,259 INFO [M:0;4618c6346924:33793 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f1f9fe63deb4888a5fed246b433b242, entries=9, sequenceid=84, filesize=6.5 K 2024-11-22T16:45:08,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:08,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44213-0x101409129ee0002, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:08,260 INFO [RS:1;4618c6346924:44213 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:45:08,260 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dadd46505e164e83a398c51e7f3e2d31 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dadd46505e164e83a398c51e7f3e2d31 2024-11-22T16:45:08,260 INFO [RS:1;4618c6346924:44213 {}] regionserver.HRegionServer(1031): Exiting; stopping=4618c6346924,44213,1732293903402; zookeeper connection closed. 2024-11-22T16:45:08,261 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@16a81869 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@16a81869 2024-11-22T16:45:08,262 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-22T16:45:08,262 INFO [M:0;4618c6346924:33793 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dadd46505e164e83a398c51e7f3e2d31, entries=2, sequenceid=84, filesize=5.1 K 2024-11-22T16:45:08,263 INFO [M:0;4618c6346924:33793 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false 2024-11-22T16:45:08,265 INFO [M:0;4618c6346924:33793 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,265 DEBUG [M:0;4618c6346924:33793 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732293908177Disabling compacts and flushes for region at 1732293908177Disabling writes for close at 1732293908178 (+1 ms)Obtaining lock to block concurrent updates at 1732293908178Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732293908178Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732293908178Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732293908179 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732293908179Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732293908201 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732293908202 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732293908205 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732293908227 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732293908227Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732293908231 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732293908251 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732293908251Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55dd33e8: reopening flushed file at 1732293908255 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@798891a2: reopening flushed file at 1732293908257 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7472a485: reopening flushed file at 1732293908260 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false at 1732293908263 (+3 ms)Writing region close event to WAL at 1732293908265 (+2 ms)Closed at 1732293908265 2024-11-22T16:45:08,267 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:45:08,267 INFO [M:0;4618c6346924:33793 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T16:45:08,267 INFO [M:0;4618c6346924:33793 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33793 2024-11-22T16:45:08,268 INFO [M:0;4618c6346924:33793 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:45:08,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:08,371 INFO [M:0;4618c6346924:33793 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:45:08,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33793-0x101409129ee0000, quorum=127.0.0.1:62922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:08,375 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T16:45:08,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T16:45:08,395 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,403 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/4618c6346924:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/4618c6346924:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732293908035 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732293907870 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/4618c6346924:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732293906344 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/4618c6346924:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732293908038 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=243 (was 229) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7716 (was 8054) 2024-11-22T16:45:08,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=243, ProcessCount=11, AvailableMemoryMB=7716 2024-11-22T16:45:08,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/hadoop-log-dir so I do NOT create it in target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d 2024-11-22T16:45:08,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/hadoop-tmp-dir so I do NOT create it in target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d 2024-11-22T16:45:08,410 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,413 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/cluster_5cf37318-ff5e-cc96-9630-fa94c9fe1be7, deleteOnExit=true 2024-11-22T16:45:08,416 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/cluster_5cf37318-ff5e-cc96-9630-fa94c9fe1be7/zookeeper_0, clientPort=62573, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/cluster_5cf37318-ff5e-cc96-9630-fa94c9fe1be7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/cluster_5cf37318-ff5e-cc96-9630-fa94c9fe1be7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T16:45:08,417 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62573 2024-11-22T16:45:08,429 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 with version=8 2024-11-22T16:45:08,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/hbase-staging 2024-11-22T16:45:08,432 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:45:08,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,432 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:45:08,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:45:08,433 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T16:45:08,433 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:45:08,433 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33939 2024-11-22T16:45:08,435 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33939 connecting to ZooKeeper ensemble=127.0.0.1:62573 2024-11-22T16:45:08,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339390x0, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:45:08,443 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33939-0x1014091424f0000 connected 2024-11-22T16:45:08,459 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,459 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:08,460 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3, hbase.cluster.distributed=false 2024-11-22T16:45:08,462 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:45:08,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33939 2024-11-22T16:45:08,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33939 2024-11-22T16:45:08,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33939 2024-11-22T16:45:08,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33939 2024-11-22T16:45:08,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33939 2024-11-22T16:45:08,480 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:45:08,481 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,481 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,481 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:45:08,481 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,481 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:45:08,481 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T16:45:08,481 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:45:08,482 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43693 2024-11-22T16:45:08,483 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43693 connecting to ZooKeeper ensemble=127.0.0.1:62573 2024-11-22T16:45:08,483 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,483 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436930x0, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:45:08,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43693-0x1014091424f0001 connected 2024-11-22T16:45:08,487 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:08,487 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T16:45:08,488 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.cache 2024-11-22T16:45:08,488 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T16:45:08,489 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T16:45:08,489 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T16:45:08,489 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T16:45:08,489 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T16:45:08,490 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T16:45:08,490 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence is complete. 2024-11-22T16:45:08,490 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T16:45:08,491 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T16:45:08,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T16:45:08,493 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:45:08,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43693 2024-11-22T16:45:08,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43693 2024-11-22T16:45:08,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43693 2024-11-22T16:45:08,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43693 2024-11-22T16:45:08,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43693 2024-11-22T16:45:08,512 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:45:08,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,513 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:45:08,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:08,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:45:08,513 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T16:45:08,513 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:45:08,514 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40897 2024-11-22T16:45:08,515 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40897 connecting to ZooKeeper ensemble=127.0.0.1:62573 2024-11-22T16:45:08,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:408970x0, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:45:08,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:08,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40897-0x1014091424f0002 connected 2024-11-22T16:45:08,520 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T16:45:08,520 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.cache 2024-11-22T16:45:08,520 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T16:45:08,520 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T16:45:08,520 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T16:45:08,521 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T16:45:08,521 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T16:45:08,521 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T16:45:08,521 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T16:45:08,521 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence is complete. 2024-11-22T16:45:08,521 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T16:45:08,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T16:45:08,523 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:45:08,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40897 2024-11-22T16:45:08,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40897 2024-11-22T16:45:08,526 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40897 2024-11-22T16:45:08,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40897 2024-11-22T16:45:08,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40897 2024-11-22T16:45:08,539 DEBUG [M:0;4618c6346924:33939 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4618c6346924:33939 2024-11-22T16:45:08,539 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4618c6346924,33939,1732293908431 2024-11-22T16:45:08,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:08,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:08,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:08,548 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4618c6346924,33939,1732293908431 2024-11-22T16:45:08,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T16:45:08,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T16:45:08,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,551 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T16:45:08,552 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4618c6346924,33939,1732293908431 from backup master directory 2024-11-22T16:45:08,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4618c6346924,33939,1732293908431 2024-11-22T16:45:08,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:08,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:08,553 WARN [master/4618c6346924:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:45:08,553 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4618c6346924,33939,1732293908431 2024-11-22T16:45:08,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:45:08,555 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/hbase.id] with ID: 05406bda-ab17-43a6-b0fb-900d4b314da5 2024-11-22T16:45:08,555 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.tmp/hbase.id 2024-11-22T16:45:08,562 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/hbase.id] 2024-11-22T16:45:08,600 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:08,600 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T16:45:08,602 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-22T16:45:08,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,609 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:45:08,610 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T16:45:08,610 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:08,634 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store 2024-11-22T16:45:08,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:08,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:45:08,641 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:45:08,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,641 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:08,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732293908641Disabling compacts and flushes for region at 1732293908641Disabling writes for close at 1732293908641Writing region close event to WAL at 1732293908641Closed at 1732293908641 2024-11-22T16:45:08,641 WARN [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/.initializing 2024-11-22T16:45:08,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/WALs/4618c6346924,33939,1732293908431 2024-11-22T16:45:08,647 INFO [master/4618c6346924:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:45:08,651 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C33939%2C1732293908431, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/WALs/4618c6346924,33939,1732293908431, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/oldWALs, maxLogs=10 2024-11-22T16:45:08,673 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/WALs/4618c6346924,33939,1732293908431/4618c6346924%2C33939%2C1732293908431.1732293908651 2024-11-22T16:45:08,674 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:08,674 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:08,674 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:08,674 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,674 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T16:45:08,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:08,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T16:45:08,684 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:08,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T16:45:08,688 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:08,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T16:45:08,692 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:08,692 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,693 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,693 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,693 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,693 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,693 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:45:08,694 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:45:08,699 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:45:08,700 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60346981, jitterRate=-0.10075990855693817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:45:08,701 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732293908675Initializing all the Stores at 1732293908675Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293908675Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293908675Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293908675Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293908675Cleaning up temporary data from old regions at 1732293908693 (+18 ms)Region opened successfully at 1732293908700 (+7 ms) 2024-11-22T16:45:08,701 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T16:45:08,705 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5f1dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:45:08,706 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T16:45:08,706 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T16:45:08,706 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T16:45:08,706 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:45:08,707 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T16:45:08,707 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-22T16:45:08,707 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T16:45:08,709 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T16:45:08,710 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T16:45:08,712 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T16:45:08,713 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T16:45:08,713 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T16:45:08,715 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T16:45:08,715 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T16:45:08,716 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T16:45:08,717 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T16:45:08,718 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T16:45:08,719 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T16:45:08,721 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T16:45:08,725 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T16:45:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,728 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4618c6346924,33939,1732293908431, sessionid=0x1014091424f0000, setting cluster-up flag (Was=false) 2024-11-22T16:45:08,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,736 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T16:45:08,738 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,33939,1732293908431 2024-11-22T16:45:08,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:08,746 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T16:45:08,747 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,33939,1732293908431 2024-11-22T16:45:08,748 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T16:45:08,749 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:08,750 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T16:45:08,750 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T16:45:08,750 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4618c6346924,33939,1732293908431 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4618c6346924:0, corePoolSize=10, maxPoolSize=10 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:45:08,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,753 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293938753 2024-11-22T16:45:08,753 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T16:45:08,753 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T16:45:08,754 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T16:45:08,754 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T16:45:08,754 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T16:45:08,754 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T16:45:08,754 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:08,754 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T16:45:08,754 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T16:45:08,754 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,754 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T16:45:08,755 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T16:45:08,755 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T16:45:08,755 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T16:45:08,755 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T16:45:08,756 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293908755,5,FailOnTimeoutGroup] 2024-11-22T16:45:08,760 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293908756,5,FailOnTimeoutGroup] 2024-11-22T16:45:08,760 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,760 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T16:45:08,760 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,760 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,769 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T16:45:08,769 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:08,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:08,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T16:45:08,779 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:45:08,779 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:08,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T16:45:08,782 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:45:08,782 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:08,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T16:45:08,786 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:45:08,786 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:08,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T16:45:08,789 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:45:08,789 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:08,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:08,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:45:08,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:08,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:08,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:45:08,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:45:08,791 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:45:08,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:45:08,797 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:45:08,797 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67684889, jitterRate=0.008583441376686096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:45:08,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732293908775Initializing all the Stores at 1732293908775Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293908775Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293908775Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293908775Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293908775Cleaning up temporary data from old regions at 1732293908790 (+15 ms)Region opened successfully at 1732293908798 (+8 ms) 2024-11-22T16:45:08,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:45:08,798 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:45:08,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:45:08,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:45:08,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:45:08,799 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:45:08,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732293908798Disabling compacts and flushes for region at 1732293908798Disabling writes for close at 1732293908798Writing region close event to WAL at 1732293908799 (+1 ms)Closed at 1732293908799 2024-11-22T16:45:08,800 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:08,800 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T16:45:08,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T16:45:08,802 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:45:08,802 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T16:45:08,829 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(746): ClusterId : 05406bda-ab17-43a6-b0fb-900d4b314da5 2024-11-22T16:45:08,829 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(746): ClusterId : 05406bda-ab17-43a6-b0fb-900d4b314da5 2024-11-22T16:45:08,829 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T16:45:08,829 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T16:45:08,837 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T16:45:08,837 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T16:45:08,837 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T16:45:08,837 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T16:45:08,840 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T16:45:08,840 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T16:45:08,841 DEBUG [RS:1;4618c6346924:40897 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e2a12c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:45:08,841 DEBUG [RS:0;4618c6346924:43693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c35878e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:45:08,853 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4618c6346924:40897 2024-11-22T16:45:08,853 INFO [RS:1;4618c6346924:40897 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T16:45:08,853 INFO [RS:1;4618c6346924:40897 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T16:45:08,853 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T16:45:08,854 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,33939,1732293908431 with port=40897, startcode=1732293908512 2024-11-22T16:45:08,854 DEBUG [RS:1;4618c6346924:40897 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T16:45:08,857 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55729, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T16:45:08,858 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33939 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4618c6346924,40897,1732293908512 2024-11-22T16:45:08,858 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33939 {}] master.ServerManager(517): Registering regionserver=4618c6346924,40897,1732293908512 2024-11-22T16:45:08,859 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4618c6346924:43693 2024-11-22T16:45:08,859 INFO [RS:0;4618c6346924:43693 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T16:45:08,859 INFO [RS:0;4618c6346924:43693 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T16:45:08,859 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T16:45:08,859 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:08,859 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T16:45:08,859 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T16:45:08,860 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,33939,1732293908431 with port=43693, startcode=1732293908480 2024-11-22T16:45:08,860 DEBUG [RS:0;4618c6346924:43693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T16:45:08,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:08,862 DEBUG [RS:1;4618c6346924:40897 {}] zookeeper.ZKUtil(111): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4618c6346924,40897,1732293908512 2024-11-22T16:45:08,862 WARN [RS:1;4618c6346924:40897 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:45:08,862 INFO [RS:1;4618c6346924:40897 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:08,862 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,40897,1732293908512 2024-11-22T16:45:08,862 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57903, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T16:45:08,863 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4618c6346924,40897,1732293908512] 2024-11-22T16:45:08,863 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33939 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4618c6346924,43693,1732293908480 2024-11-22T16:45:08,863 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33939 {}] master.ServerManager(517): Registering regionserver=4618c6346924,43693,1732293908480 2024-11-22T16:45:08,864 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:08,865 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T16:45:08,865 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T16:45:08,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:08,872 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4618c6346924,43693,1732293908480] 2024-11-22T16:45:08,872 DEBUG [RS:0;4618c6346924:43693 {}] zookeeper.ZKUtil(111): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4618c6346924,43693,1732293908480 2024-11-22T16:45:08,872 WARN [RS:0;4618c6346924:43693 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:45:08,872 INFO [RS:0;4618c6346924:43693 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:08,872 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,43693,1732293908480 2024-11-22T16:45:08,873 INFO [RS:1;4618c6346924:40897 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T16:45:08,877 INFO [RS:0;4618c6346924:43693 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T16:45:08,880 INFO [RS:1;4618c6346924:40897 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T16:45:08,880 INFO [RS:0;4618c6346924:43693 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T16:45:08,881 INFO [RS:1;4618c6346924:40897 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T16:45:08,881 INFO [RS:0;4618c6346924:43693 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T16:45:08,881 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,881 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,882 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T16:45:08,882 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T16:45:08,882 INFO [RS:1;4618c6346924:40897 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T16:45:08,882 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,882 INFO [RS:0;4618c6346924:43693 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T16:45:08,883 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,883 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:45:08,883 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,883 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:45:08,884 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:08,884 DEBUG [RS:1;4618c6346924:40897 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:08,884 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:08,885 DEBUG [RS:0;4618c6346924:43693 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:08,885 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,40897,1732293908512-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,886 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,43693,1732293908480-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:45:08,902 INFO [RS:0;4618c6346924:43693 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T16:45:08,902 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,43693,1732293908480-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,902 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,902 INFO [RS:0;4618c6346924:43693 {}] regionserver.Replication(171): 4618c6346924,43693,1732293908480 started 2024-11-22T16:45:08,902 INFO [RS:1;4618c6346924:40897 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T16:45:08,902 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,40897,1732293908512-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,902 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,903 INFO [RS:1;4618c6346924:40897 {}] regionserver.Replication(171): 4618c6346924,40897,1732293908512 started 2024-11-22T16:45:08,917 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,918 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1482): Serving as 4618c6346924,40897,1732293908512, RpcServer on 4618c6346924/172.17.0.2:40897, sessionid=0x1014091424f0002 2024-11-22T16:45:08,918 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T16:45:08,918 DEBUG [RS:1;4618c6346924:40897 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4618c6346924,40897,1732293908512 2024-11-22T16:45:08,918 DEBUG [RS:1;4618c6346924:40897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,40897,1732293908512' 2024-11-22T16:45:08,918 DEBUG [RS:1;4618c6346924:40897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T16:45:08,919 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:08,919 DEBUG [RS:1;4618c6346924:40897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T16:45:08,919 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1482): Serving as 4618c6346924,43693,1732293908480, RpcServer on 4618c6346924/172.17.0.2:43693, sessionid=0x1014091424f0001 2024-11-22T16:45:08,919 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T16:45:08,919 DEBUG [RS:0;4618c6346924:43693 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4618c6346924,43693,1732293908480 2024-11-22T16:45:08,919 DEBUG [RS:0;4618c6346924:43693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,43693,1732293908480' 2024-11-22T16:45:08,919 DEBUG [RS:0;4618c6346924:43693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T16:45:08,919 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T16:45:08,919 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T16:45:08,920 DEBUG [RS:1;4618c6346924:40897 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4618c6346924,40897,1732293908512 2024-11-22T16:45:08,920 DEBUG [RS:1;4618c6346924:40897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,40897,1732293908512' 2024-11-22T16:45:08,920 DEBUG [RS:1;4618c6346924:40897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T16:45:08,920 DEBUG [RS:0;4618c6346924:43693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T16:45:08,920 DEBUG [RS:1;4618c6346924:40897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T16:45:08,920 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T16:45:08,920 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T16:45:08,920 DEBUG [RS:0;4618c6346924:43693 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4618c6346924,43693,1732293908480 2024-11-22T16:45:08,920 DEBUG [RS:0;4618c6346924:43693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,43693,1732293908480' 2024-11-22T16:45:08,920 DEBUG [RS:0;4618c6346924:43693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T16:45:08,921 DEBUG [RS:1;4618c6346924:40897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T16:45:08,921 INFO [RS:1;4618c6346924:40897 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T16:45:08,921 INFO [RS:1;4618c6346924:40897 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T16:45:08,921 DEBUG [RS:0;4618c6346924:43693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T16:45:08,921 DEBUG [RS:0;4618c6346924:43693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T16:45:08,921 INFO [RS:0;4618c6346924:43693 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T16:45:08,921 INFO [RS:0;4618c6346924:43693 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T16:45:08,953 WARN [4618c6346924:33939 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:45:09,022 INFO [RS:1;4618c6346924:40897 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:45:09,022 INFO [RS:0;4618c6346924:43693 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:45:09,023 INFO [RS:1;4618c6346924:40897 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4618c6346924%2C40897%2C1732293908512, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,40897,1732293908512, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs, maxLogs=55 2024-11-22T16:45:09,023 INFO [RS:0;4618c6346924:43693 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4618c6346924%2C43693%2C1732293908480, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,43693,1732293908480, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs, maxLogs=55 2024-11-22T16:45:09,045 INFO [RS:1;4618c6346924:40897 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,40897,1732293908512/4618c6346924%2C40897%2C1732293908512.1732293909025 2024-11-22T16:45:09,045 INFO [RS:0;4618c6346924:43693 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,43693,1732293908480/4618c6346924%2C43693%2C1732293908480.1732293909025 2024-11-22T16:45:09,045 DEBUG [RS:1;4618c6346924:40897 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:09,045 DEBUG [RS:0;4618c6346924:43693 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:09,203 DEBUG [4618c6346924:33939 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T16:45:09,204 DEBUG [4618c6346924:33939 {}] balancer.BalancerClusterState(204): Hosts are {4618c6346924=0} racks are {/default-rack=0} 2024-11-22T16:45:09,204 DEBUG [4618c6346924:33939 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T16:45:09,204 DEBUG [4618c6346924:33939 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T16:45:09,204 DEBUG [4618c6346924:33939 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T16:45:09,204 DEBUG [4618c6346924:33939 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T16:45:09,204 INFO [4618c6346924:33939 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T16:45:09,204 INFO [4618c6346924:33939 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T16:45:09,204 DEBUG [4618c6346924:33939 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T16:45:09,205 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4618c6346924,40897,1732293908512 2024-11-22T16:45:09,206 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,40897,1732293908512, state=OPENING 2024-11-22T16:45:09,208 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T16:45:09,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:09,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:09,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:09,210 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:45:09,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:09,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:09,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:09,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,40897,1732293908512}] 2024-11-22T16:45:09,363 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T16:45:09,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43225, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T16:45:09,368 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T16:45:09,368 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:09,369 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T16:45:09,370 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4618c6346924%2C40897%2C1732293908512.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,40897,1732293908512, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs, maxLogs=55 2024-11-22T16:45:09,393 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,40897,1732293908512/4618c6346924%2C40897%2C1732293908512.meta.1732293909371.meta 2024-11-22T16:45:09,393 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:09,394 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:09,394 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T16:45:09,394 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T16:45:09,394 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T16:45:09,394 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T16:45:09,394 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:09,395 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T16:45:09,395 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T16:45:09,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T16:45:09,396 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:45:09,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:09,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:09,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T16:45:09,397 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:45:09,397 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:09,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:09,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T16:45:09,398 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:45:09,398 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:09,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:09,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T16:45:09,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:45:09,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:09,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:45:09,399 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:45:09,400 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:09,400 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740 2024-11-22T16:45:09,400 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:45:09,400 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:45:09,400 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:45:09,401 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:45:09,402 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64299325, jitterRate=-0.04186539351940155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:45:09,402 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T16:45:09,403 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732293909395Writing region info on filesystem at 1732293909395Initializing all the Stores at 1732293909395Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293909395Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293909395Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293909395Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293909395Cleaning up temporary data from old regions at 1732293909400 (+5 ms)Running coprocessor post-open hooks at 1732293909402 (+2 ms)Region opened successfully at 1732293909403 (+1 ms) 2024-11-22T16:45:09,404 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732293909363 2024-11-22T16:45:09,406 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T16:45:09,407 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T16:45:09,407 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,40897,1732293908512 2024-11-22T16:45:09,408 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,40897,1732293908512, state=OPEN 2024-11-22T16:45:09,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:45:09,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:45:09,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:45:09,410 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4618c6346924,40897,1732293908512 2024-11-22T16:45:09,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:09,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:09,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:45:09,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T16:45:09,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,40897,1732293908512 in 200 msec 2024-11-22T16:45:09,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T16:45:09,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec 2024-11-22T16:45:09,414 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:45:09,414 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T16:45:09,418 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:45:09,418 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,40897,1732293908512, seqNum=-1] 2024-11-22T16:45:09,418 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:45:09,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47929, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:45:09,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 676 msec 2024-11-22T16:45:09,426 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732293909426, completionTime=-1 2024-11-22T16:45:09,426 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T16:45:09,426 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:45:09,428 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T16:45:09,428 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293969428 2024-11-22T16:45:09,428 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732294029428 2024-11-22T16:45:09,428 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-22T16:45:09,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33939,1732293908431-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:09,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33939,1732293908431-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:09,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33939,1732293908431-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:09,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4618c6346924:33939, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:09,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:09,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:09,432 DEBUG [master/4618c6346924:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T16:45:09,434 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.880sec 2024-11-22T16:45:09,434 INFO [master/4618c6346924:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T16:45:09,434 INFO [master/4618c6346924:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T16:45:09,435 INFO [master/4618c6346924:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T16:45:09,435 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T16:45:09,435 INFO [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T16:45:09,435 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33939,1732293908431-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:45:09,435 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33939,1732293908431-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T16:45:09,436 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T16:45:09,436 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T16:45:09,436 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,33939,1732293908431-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:09,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:45:09,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4618c6346924,33939,-1 for getting cluster id 2024-11-22T16:45:09,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T16:45:09,532 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '05406bda-ab17-43a6-b0fb-900d4b314da5' 2024-11-22T16:45:09,532 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T16:45:09,532 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "05406bda-ab17-43a6-b0fb-900d4b314da5" 2024-11-22T16:45:09,533 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc7f4fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:45:09,533 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4618c6346924,33939,-1] 2024-11-22T16:45:09,533 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T16:45:09,534 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:09,535 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T16:45:09,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:45:09,537 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:45:09,539 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,40897,1732293908512, seqNum=-1] 2024-11-22T16:45:09,539 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:45:09,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:45:09,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4618c6346924,33939,1732293908431 2024-11-22T16:45:09,544 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T16:45:09,546 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 4618c6346924,33939,1732293908431 2024-11-22T16:45:09,546 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b701d0b 2024-11-22T16:45:09,546 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T16:45:09,548 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T16:45:09,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33939 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732293909544', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:45:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33939 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 2024-11-22T16:45:09,551 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:45:09,551 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:09,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33939 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732293909544" procId is: 4 2024-11-22T16:45:09,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:45:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:09,564 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff67d88bc2134e831f87243685e5003a, NAME => 'testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732293909544', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:09,571 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:09,571 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(1722): Closing ff67d88bc2134e831f87243685e5003a, disabling compactions & flushes 2024-11-22T16:45:09,571 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:09,571 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:09,571 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. after waiting 0 ms 2024-11-22T16:45:09,571 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:09,571 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:09,571 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732293909544-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff67d88bc2134e831f87243685e5003a: Waiting for close lock at 1732293909571Disabling compacts and flushes for region at 1732293909571Disabling writes for close at 1732293909571Writing region close event to WAL at 1732293909571Closed at 1732293909571 2024-11-22T16:45:09,573 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:45:09,573 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732293909573"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293909573"}]},"ts":"1732293909573"} 2024-11-22T16:45:09,576 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:45:09,577 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:45:09,577 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732293909544","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293909577"}]},"ts":"1732293909577"} 2024-11-22T16:45:09,579 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732293909544, state=ENABLING in hbase:meta 2024-11-22T16:45:09,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4618c6346924=0} racks are {/default-rack=0} 2024-11-22T16:45:09,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T16:45:09,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T16:45:09,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T16:45:09,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T16:45:09,580 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T16:45:09,580 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T16:45:09,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T16:45:09,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN}] 2024-11-22T16:45:09,582 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN 2024-11-22T16:45:09,582 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN; state=OFFLINE, location=4618c6346924,43693,1732293908480; forceNewPlan=false, retain=false 2024-11-22T16:45:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:09,733 INFO [4618c6346924:33939 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T16:45:09,733 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff67d88bc2134e831f87243685e5003a, regionState=OPENING, regionLocation=4618c6346924,43693,1732293908480 2024-11-22T16:45:09,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN because future has completed 2024-11-22T16:45:09,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff67d88bc2134e831f87243685e5003a, server=4618c6346924,43693,1732293908480}] 2024-11-22T16:45:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:09,890 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T16:45:09,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35919, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T16:45:09,894 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:09,894 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ff67d88bc2134e831f87243685e5003a, NAME => 'testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:09,895 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732293909544 ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,895 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:09,895 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,895 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,896 INFO [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,900 INFO [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff67d88bc2134e831f87243685e5003a columnFamilyName family 2024-11-22T16:45:09,900 DEBUG [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:09,900 INFO [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] regionserver.HStore(327): Store=ff67d88bc2134e831f87243685e5003a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:09,900 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,901 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,901 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,901 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,901 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,902 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,911 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:45:09,911 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ff67d88bc2134e831f87243685e5003a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71369719, jitterRate=0.06349168717861176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:45:09,911 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:09,912 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ff67d88bc2134e831f87243685e5003a: Running coprocessor pre-open hook at 1732293909895Writing region info on filesystem at 1732293909895Initializing all the Stores at 1732293909895Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293909895Cleaning up temporary data from old regions at 1732293909901 (+6 ms)Running coprocessor post-open hooks at 1732293909911 (+10 ms)Region opened successfully at 1732293909912 (+1 ms) 2024-11-22T16:45:09,913 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a., pid=6, masterSystemTime=1732293909889 2024-11-22T16:45:09,915 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:09,916 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:09,916 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff67d88bc2134e831f87243685e5003a, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,43693,1732293908480 2024-11-22T16:45:09,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff67d88bc2134e831f87243685e5003a, server=4618c6346924,43693,1732293908480 because future has completed 2024-11-22T16:45:09,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T16:45:09,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ff67d88bc2134e831f87243685e5003a, server=4618c6346924,43693,1732293908480 in 183 msec 2024-11-22T16:45:09,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T16:45:09,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN in 342 msec 2024-11-22T16:45:09,925 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:45:09,925 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732293909544","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293909925"}]},"ts":"1732293909925"} 2024-11-22T16:45:09,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732293909544, state=ENABLED in hbase:meta 2024-11-22T16:45:09,928 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:45:09,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732293909544 in 379 msec 2024-11-22T16:45:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:45:10,181 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732293909544 completed 2024-11-22T16:45:10,181 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732293909544 get assigned. Timeout = 60000ms 2024-11-22T16:45:10,181 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:45:10,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732293909544 assigned to meta. Checking AM states. 2024-11-22T16:45:10,185 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:45:10,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732293909544 assigned. 2024-11-22T16:45:10,189 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732293909544', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a., hostname=4618c6346924,43693,1732293908480, seqNum=2] 2024-11-22T16:45:10,189 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:45:10,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:45:10,195 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ff67d88bc2134e831f87243685e5003a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T16:45:10,205 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ccee317ea34247daa164ed2c7b26fa51 2024-11-22T16:45:10,218 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/.tmp/family/ccee317ea34247daa164ed2c7b26fa51 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T16:45:10,219 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/.tmp/family/ccee317ea34247daa164ed2c7b26fa51 2024-11-22T16:45:10,221 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/.tmp/family/ccee317ea34247daa164ed2c7b26fa51 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51 2024-11-22T16:45:10,223 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51, delay=974 ms 2024-11-22T16:45:10,224 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T16:45:10,225 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ff67d88bc2134e831f87243685e5003a in 30ms, sequenceid=6, compaction requested=false 2024-11-22T16:45:10,225 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732293909544' 2024-11-22T16:45:10,225 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ff67d88bc2134e831f87243685e5003a: 2024-11-22T16:45:10,695 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:45:11,198 DEBUG [hfile-prefetch-1732293907870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ccee317ea34247daa164ed2c7b26fa51_0. 2024-11-22T16:45:11,198 DEBUG [hfile-prefetch-1732293907870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ccee317ea34247daa164ed2c7b26fa51_117. 2024-11-22T16:45:11,199 DEBUG [hfile-prefetch-1732293907870 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51, with total blocks 2, and data blocks 1 2024-11-22T16:45:11,199 DEBUG [hfile-prefetch-1732293907870 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T16:45:11,199 DEBUG [hfile-prefetch-1732293907870 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T16:45:11,199 DEBUG [hfile-prefetch-1732293907870 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51. Will sleep 100ms and try the verification again. 2024-11-22T16:45:11,223 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T16:45:11,223 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T16:45:11,223 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:11,223 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:11,226 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;4618c6346924:43693,5,FailOnTimeoutGroup] 2024-11-22T16:45:11,226 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4618c6346924,43693,1732293908480' ***** 2024-11-22T16:45:11,226 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-22T16:45:11,226 INFO [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T16:45:11,226 INFO [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence 2024-11-22T16:45:11,226 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T16:45:11,226 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T16:45:11,226 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T16:45:11,226 INFO [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T16:45:11,227 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T16:45:11,227 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732293911223 2024-11-22T16:45:11,227 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T16:45:11,227 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T16:45:11,237 DEBUG [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1487): Thread RS:0;4618c6346924:43693 started persisting bucket cache to file 2024-11-22T16:45:11,237 DEBUG [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:11,237 DEBUG [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:11,244 DEBUG [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T16:45:11,244 DEBUG [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732293911237 2024-11-22T16:45:11,244 DEBUG [RS:0;4618c6346924:43693 {}] bucket.BucketCache(1509): Thread RS:0;4618c6346924:43693 finished persisting bucket cache to file, renaming 2024-11-22T16:45:11,245 INFO [RS:0;4618c6346924:43693 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T16:45:11,245 INFO [RS:0;4618c6346924:43693 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T16:45:11,245 INFO [RS:0;4618c6346924:43693 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T16:45:11,245 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T16:45:11,245 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(3091): Received CLOSE for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,245 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(959): stopping server 4618c6346924,43693,1732293908480 2024-11-22T16:45:11,245 INFO [RS:0;4618c6346924:43693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:45:11,245 INFO [RS:0;4618c6346924:43693 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4618c6346924:43693. 2024-11-22T16:45:11,245 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T16:45:11,245 DEBUG [RS:0;4618c6346924:43693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:11,245 DEBUG [RS:0;4618c6346924:43693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:11,245 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff67d88bc2134e831f87243685e5003a, disabling compactions & flushes 2024-11-22T16:45:11,245 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,246 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,246 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T16:45:11,246 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. after waiting 0 ms 2024-11-22T16:45:11,246 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1325): Online Regions={ff67d88bc2134e831f87243685e5003a=testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.} 2024-11-22T16:45:11,246 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,246 DEBUG [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1351): Waiting on ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,246 DEBUG [StoreFileCloser-ff67d88bc2134e831f87243685e5003a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51 2024-11-22T16:45:11,246 WARN [hfile-prefetch-1732293907870 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T16:45:11,247 DEBUG [hfile-prefetch-1732293907870 {}] hfile.PrefetchExecutor(121): Prefetch completed for ccee317ea34247daa164ed2c7b26fa51 2024-11-22T16:45:11,251 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T16:45:11,252 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,252 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff67d88bc2134e831f87243685e5003a: Waiting for close lock at 1732293911245Running coprocessor pre-close hooks at 1732293911245Disabling compacts and flushes for region at 1732293911245Disabling writes for close at 1732293911246 (+1 ms)Writing region close event to WAL at 1732293911247 (+1 ms)Running coprocessor post-close hooks at 1732293911252 (+5 ms)Closed at 1732293911252 2024-11-22T16:45:11,252 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,288 INFO [regionserver/4618c6346924:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:11,446 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(976): stopping server 4618c6346924,43693,1732293908480; all regions closed. 2024-11-22T16:45:11,448 DEBUG [RS:0;4618c6346924:43693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs 2024-11-22T16:45:11,448 INFO [RS:0;4618c6346924:43693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C43693%2C1732293908480:(num 1732293909025) 2024-11-22T16:45:11,448 DEBUG [RS:0;4618c6346924:43693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:11,448 INFO [RS:0;4618c6346924:43693 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:11,448 INFO [RS:0;4618c6346924:43693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:45:11,448 INFO [RS:0;4618c6346924:43693 {}] hbase.ChoreService(370): Chore service for: regionserver/4618c6346924:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T16:45:11,449 INFO [RS:0;4618c6346924:43693 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T16:45:11,449 INFO [RS:0;4618c6346924:43693 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T16:45:11,449 INFO [RS:0;4618c6346924:43693 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T16:45:11,449 INFO [RS:0;4618c6346924:43693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:45:11,449 INFO [regionserver/4618c6346924:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:45:11,449 INFO [RS:0;4618c6346924:43693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43693 2024-11-22T16:45:11,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:11,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4618c6346924,43693,1732293908480 2024-11-22T16:45:11,453 INFO [RS:0;4618c6346924:43693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:45:11,454 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4618c6346924,43693,1732293908480] 2024-11-22T16:45:11,457 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4618c6346924,43693,1732293908480 already deleted, retry=false 2024-11-22T16:45:11,457 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 4618c6346924,43693,1732293908480 on 4618c6346924,33939,1732293908431 2024-11-22T16:45:11,462 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 4618c6346924,43693,1732293908480, splitWal=true, meta=false 2024-11-22T16:45:11,464 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 4618c6346924,43693,1732293908480, splitWal=true, meta=false 2024-11-22T16:45:11,464 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 4618c6346924,43693,1732293908480 (carryingMeta=false) 4618c6346924,43693,1732293908480/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6ff313f0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-22T16:45:11,465 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 4618c6346924,43693,1732293908480 had 1 regions 2024-11-22T16:45:11,466 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 4618c6346924,43693,1732293908480, splitWal=true, meta=false, isMeta: false 2024-11-22T16:45:11,467 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,43693,1732293908480-splitting 2024-11-22T16:45:11,467 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,43693,1732293908480-splitting dir is empty, no logs to split. 2024-11-22T16:45:11,467 INFO [PEWorker-2 {}] master.SplitWALManager(105): 4618c6346924,43693,1732293908480 WAL count=0, meta=false 2024-11-22T16:45:11,470 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,43693,1732293908480-splitting dir is empty, no logs to split. 2024-11-22T16:45:11,470 INFO [PEWorker-2 {}] master.SplitWALManager(105): 4618c6346924,43693,1732293908480 WAL count=0, meta=false 2024-11-22T16:45:11,470 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 4618c6346924,43693,1732293908480 WAL splitting is done? wals=0, meta=false 2024-11-22T16:45:11,470 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 4618c6346924,43693,1732293908480 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,43693,1732293908480-splitting does not exist 2024-11-22T16:45:11,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN}] 2024-11-22T16:45:11,473 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN 2024-11-22T16:45:11,474 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T16:45:11,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:11,556 INFO [RS:0;4618c6346924:43693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:45:11,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43693-0x1014091424f0001, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:11,556 INFO [RS:0;4618c6346924:43693 {}] regionserver.HRegionServer(1031): Exiting; stopping=4618c6346924,43693,1732293908480; zookeeper connection closed. 2024-11-22T16:45:11,557 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@369cfcf7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@369cfcf7 2024-11-22T16:45:11,624 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff67d88bc2134e831f87243685e5003a, regionState=OPENING, regionLocation=4618c6346924,40897,1732293908512 2024-11-22T16:45:11,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN because future has completed 2024-11-22T16:45:11,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff67d88bc2134e831f87243685e5003a, server=4618c6346924,40897,1732293908512}] 2024-11-22T16:45:11,743 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:45:11,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:11,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:11,743 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:45:11,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:45:11,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:45:11,743 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T16:45:11,744 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:45:11,744 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34043 2024-11-22T16:45:11,746 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34043 connecting to ZooKeeper ensemble=127.0.0.1:62573 2024-11-22T16:45:11,746 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:11,746 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:11,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340430x0, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:45:11,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34043-0x1014091424f0003 connected 2024-11-22T16:45:11,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-22T16:45:11,750 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-22T16:45:11,750 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T16:45:11,751 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.cache 2024-11-22T16:45:11,751 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T16:45:11,751 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T16:45:11,751 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T16:45:11,752 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T16:45:11,752 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T16:45:11,752 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T16:45:11,753 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-22T16:45:11,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T16:45:11,754 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-22T16:45:11,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:45:11,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34043 2024-11-22T16:45:11,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34043 2024-11-22T16:45:11,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34043 2024-11-22T16:45:11,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34043 2024-11-22T16:45:11,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34043 2024-11-22T16:45:11,757 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-22T16:45:11,757 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T16:45:11,758 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-22T16:45:11,758 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence is complete. 2024-11-22T16:45:11,758 INFO [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(746): ClusterId : 05406bda-ab17-43a6-b0fb-900d4b314da5 2024-11-22T16:45:11,758 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T16:45:11,761 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T16:45:11,761 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T16:45:11,763 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T16:45:11,764 DEBUG [RS:2;4618c6346924:34043 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb95b2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:45:11,776 DEBUG [RS:2;4618c6346924:34043 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;4618c6346924:34043 2024-11-22T16:45:11,776 INFO [RS:2;4618c6346924:34043 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T16:45:11,776 INFO [RS:2;4618c6346924:34043 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T16:45:11,776 DEBUG [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T16:45:11,777 INFO [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,33939,1732293908431 with port=34043, startcode=1732293911743 2024-11-22T16:45:11,777 DEBUG [RS:2;4618c6346924:34043 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T16:45:11,779 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55481, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T16:45:11,780 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33939 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4618c6346924,34043,1732293911743 2024-11-22T16:45:11,780 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33939 {}] master.ServerManager(517): Registering regionserver=4618c6346924,34043,1732293911743 2024-11-22T16:45:11,781 DEBUG [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3 2024-11-22T16:45:11,782 DEBUG [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T16:45:11,782 DEBUG [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T16:45:11,782 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,782 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ff67d88bc2134e831f87243685e5003a, NAME => 'testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:45:11,783 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732293909544 ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,783 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:45:11,783 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,783 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:11,784 INFO [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,784 DEBUG [RS:2;4618c6346924:34043 {}] zookeeper.ZKUtil(111): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4618c6346924,34043,1732293911743 2024-11-22T16:45:11,784 WARN [RS:2;4618c6346924:34043 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:45:11,784 INFO [RS:2;4618c6346924:34043 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:45:11,785 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4618c6346924,34043,1732293911743] 2024-11-22T16:45:11,785 DEBUG [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,34043,1732293911743 2024-11-22T16:45:11,785 INFO [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff67d88bc2134e831f87243685e5003a columnFamilyName family 2024-11-22T16:45:11,785 DEBUG [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:45:11,787 DEBUG [StoreFileOpener-ff67d88bc2134e831f87243685e5003a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51, delay=911 ms 2024-11-22T16:45:11,789 DEBUG [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51 2024-11-22T16:45:11,789 INFO [StoreOpener-ff67d88bc2134e831f87243685e5003a-1 {}] regionserver.HStore(327): Store=ff67d88bc2134e831f87243685e5003a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:45:11,789 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,790 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,790 INFO [RS:2;4618c6346924:34043 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T16:45:11,791 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,791 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,791 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,792 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,793 INFO [RS:2;4618c6346924:34043 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T16:45:11,793 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ff67d88bc2134e831f87243685e5003a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72406506, jitterRate=0.07894101738929749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:45:11,793 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:11,793 INFO [RS:2;4618c6346924:34043 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T16:45:11,793 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,793 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ff67d88bc2134e831f87243685e5003a: Running coprocessor pre-open hook at 1732293911783Writing region info on filesystem at 1732293911783Initializing all the Stores at 1732293911784 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293911784Cleaning up temporary data from old regions at 1732293911791 (+7 ms)Running coprocessor post-open hooks at 1732293911793 (+2 ms)Region opened successfully at 1732293911793 2024-11-22T16:45:11,794 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a., pid=9, masterSystemTime=1732293911780 2024-11-22T16:45:11,795 INFO [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T16:45:11,796 INFO [RS:2;4618c6346924:34043 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T16:45:11,796 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,796 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,797 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,797 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,797 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:45:11,797 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:11,797 DEBUG [RS:2;4618c6346924:34043 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:45:11,797 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,797 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:11,798 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff67d88bc2134e831f87243685e5003a, regionState=OPEN, openSeqNum=10, regionLocation=4618c6346924,40897,1732293908512 2024-11-22T16:45:11,799 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,799 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,799 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,800 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,800 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,800 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,34043,1732293911743-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:45:11,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff67d88bc2134e831f87243685e5003a, server=4618c6346924,40897,1732293908512 because future has completed 2024-11-22T16:45:11,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T16:45:11,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ff67d88bc2134e831f87243685e5003a, server=4618c6346924,40897,1732293908512 in 173 msec 2024-11-22T16:45:11,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T16:45:11,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732293909544, region=ff67d88bc2134e831f87243685e5003a, ASSIGN in 331 msec 2024-11-22T16:45:11,806 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 4618c6346924,43693,1732293908480 after splitting done 2024-11-22T16:45:11,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 4618c6346924,43693,1732293908480, splitWal=true, meta=false in 348 msec 2024-11-22T16:45:11,819 INFO [RS:2;4618c6346924:34043 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T16:45:11,819 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,34043,1732293911743-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,819 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,819 INFO [RS:2;4618c6346924:34043 {}] regionserver.Replication(171): 4618c6346924,34043,1732293911743 started 2024-11-22T16:45:11,835 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:45:11,835 INFO [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(1482): Serving as 4618c6346924,34043,1732293911743, RpcServer on 4618c6346924/172.17.0.2:34043, sessionid=0x1014091424f0003 2024-11-22T16:45:11,835 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T16:45:11,835 DEBUG [RS:2;4618c6346924:34043 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4618c6346924,34043,1732293911743 2024-11-22T16:45:11,835 DEBUG [RS:2;4618c6346924:34043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,34043,1732293911743' 2024-11-22T16:45:11,835 DEBUG [RS:2;4618c6346924:34043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T16:45:11,836 DEBUG [RS:2;4618c6346924:34043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T16:45:11,837 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T16:45:11,837 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T16:45:11,837 DEBUG [RS:2;4618c6346924:34043 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4618c6346924,34043,1732293911743 2024-11-22T16:45:11,837 DEBUG [RS:2;4618c6346924:34043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,34043,1732293911743' 2024-11-22T16:45:11,837 DEBUG [RS:2;4618c6346924:34043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T16:45:11,837 DEBUG [RS:2;4618c6346924:34043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T16:45:11,838 DEBUG [RS:2;4618c6346924:34043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T16:45:11,838 INFO [RS:2;4618c6346924:34043 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T16:45:11,838 INFO [RS:2;4618c6346924:34043 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T16:45:11,938 INFO [RS:2;4618c6346924:34043 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:45:11,939 INFO [RS:2;4618c6346924:34043 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4618c6346924%2C34043%2C1732293911743, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,34043,1732293911743, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs, maxLogs=55 2024-11-22T16:45:11,959 INFO [RS:2;4618c6346924:34043 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/WALs/4618c6346924,34043,1732293911743/4618c6346924%2C34043%2C1732293911743.1732293911940 2024-11-22T16:45:11,960 DEBUG [RS:2;4618c6346924:34043 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T16:45:12,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T16:45:12,336 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:45:12,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:12,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:12,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:12,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T16:45:12,336 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T16:45:12,336 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=533830663, stopped=false 2024-11-22T16:45:12,337 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4618c6346924,33939,1732293908431 2024-11-22T16:45:12,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:12,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:12,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:45:12,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:12,339 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:45:12,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:12,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:12,339 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:45:12,339 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:12,339 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:12,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:12,339 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:12,339 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:45:12,340 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4618c6346924,40897,1732293908512' ***** 2024-11-22T16:45:12,340 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T16:45:12,340 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4618c6346924,34043,1732293911743' ***** 2024-11-22T16:45:12,340 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T16:45:12,340 INFO [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T16:45:12,340 INFO [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence 2024-11-22T16:45:12,340 INFO [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T16:45:12,340 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T16:45:12,340 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T16:45:12,340 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T16:45:12,340 INFO [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fc1b3379-dd14-e677-5449-6200e919aa2d/bucket.persistence 2024-11-22T16:45:12,340 INFO [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T16:45:12,340 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T16:45:12,340 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T16:45:12,341 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T16:45:12,341 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T16:45:12,341 INFO [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T16:45:12,341 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T16:45:12,351 DEBUG [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1487): Thread RS:1;4618c6346924:40897 started persisting bucket cache to file 2024-11-22T16:45:12,351 DEBUG [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1487): Thread RS:2;4618c6346924:34043 started persisting bucket cache to file 2024-11-22T16:45:12,351 DEBUG [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:12,351 DEBUG [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:12,351 DEBUG [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T16:45:12,351 DEBUG [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T16:45:12,354 DEBUG [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T16:45:12,355 DEBUG [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732293912351 2024-11-22T16:45:12,355 DEBUG [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T16:45:12,355 DEBUG [RS:1;4618c6346924:40897 {}] bucket.BucketCache(1509): Thread RS:1;4618c6346924:40897 finished persisting bucket cache to file, renaming 2024-11-22T16:45:12,355 DEBUG [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732293912351 2024-11-22T16:45:12,355 DEBUG [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1509): Thread RS:2;4618c6346924:34043 finished persisting bucket cache to file, renaming 2024-11-22T16:45:12,355 WARN [RS:2;4618c6346924:34043 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-22T16:45:12,355 INFO [RS:1;4618c6346924:40897 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T16:45:12,355 INFO [RS:2;4618c6346924:34043 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T16:45:12,355 INFO [RS:1;4618c6346924:40897 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T16:45:12,355 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T16:45:12,355 INFO [RS:2;4618c6346924:34043 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T16:45:12,355 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T16:45:12,355 INFO [RS:1;4618c6346924:40897 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T16:45:12,355 INFO [RS:2;4618c6346924:34043 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T16:45:12,355 INFO [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(959): stopping server 4618c6346924,34043,1732293911743 2024-11-22T16:45:12,355 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(3091): Received CLOSE for ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:12,355 INFO [RS:2;4618c6346924:34043 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:45:12,355 INFO [RS:2;4618c6346924:34043 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;4618c6346924:34043. 2024-11-22T16:45:12,355 DEBUG [RS:2;4618c6346924:34043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:12,355 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(959): stopping server 4618c6346924,40897,1732293908512 2024-11-22T16:45:12,355 DEBUG [RS:2;4618c6346924:34043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:12,355 INFO [RS:1;4618c6346924:40897 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:45:12,356 INFO [RS:1;4618c6346924:40897 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4618c6346924:40897. 2024-11-22T16:45:12,356 DEBUG [RS:1;4618c6346924:40897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:45:12,356 DEBUG [RS:1;4618c6346924:40897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:12,356 INFO [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(976): stopping server 4618c6346924,34043,1732293911743; all regions closed. 2024-11-22T16:45:12,356 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T16:45:12,356 INFO [RS:1;4618c6346924:40897 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T16:45:12,356 INFO [RS:1;4618c6346924:40897 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T16:45:12,356 INFO [RS:1;4618c6346924:40897 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T16:45:12,356 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff67d88bc2134e831f87243685e5003a, disabling compactions & flushes 2024-11-22T16:45:12,356 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T16:45:12,356 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:12,356 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:12,356 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. after waiting 0 ms 2024-11-22T16:45:12,356 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:12,356 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T16:45:12,356 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ff67d88bc2134e831f87243685e5003a=testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a.} 2024-11-22T16:45:12,356 DEBUG [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ff67d88bc2134e831f87243685e5003a 2024-11-22T16:45:12,356 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T16:45:12,356 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:45:12,356 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:45:12,357 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:45:12,357 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:45:12,357 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:45:12,357 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-22T16:45:12,358 DEBUG [RS:2;4618c6346924:34043 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs 2024-11-22T16:45:12,358 INFO [RS:2;4618c6346924:34043 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C34043%2C1732293911743:(num 1732293911940) 2024-11-22T16:45:12,358 DEBUG [RS:2;4618c6346924:34043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:12,358 INFO [RS:2;4618c6346924:34043 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:12,358 INFO [RS:2;4618c6346924:34043 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:45:12,358 INFO [RS:2;4618c6346924:34043 {}] hbase.ChoreService(370): Chore service for: regionserver/4618c6346924:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T16:45:12,359 INFO [RS:2;4618c6346924:34043 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T16:45:12,359 INFO [regionserver/4618c6346924:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:45:12,359 INFO [RS:2;4618c6346924:34043 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T16:45:12,359 INFO [RS:2;4618c6346924:34043 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T16:45:12,359 INFO [RS:2;4618c6346924:34043 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:45:12,359 INFO [RS:2;4618c6346924:34043 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34043 2024-11-22T16:45:12,359 DEBUG [StoreFileCloser-ff67d88bc2134e831f87243685e5003a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/family/ccee317ea34247daa164ed2c7b26fa51 2024-11-22T16:45:12,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4618c6346924,34043,1732293911743 2024-11-22T16:45:12,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:12,361 INFO [RS:2;4618c6346924:34043 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:45:12,365 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4618c6346924,34043,1732293911743] 2024-11-22T16:45:12,365 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/default/testBlockEvictionOnGracefulStop1732293909544/ff67d88bc2134e831f87243685e5003a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T16:45:12,365 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:12,365 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff67d88bc2134e831f87243685e5003a: Waiting for close lock at 1732293912356Running coprocessor pre-close hooks at 1732293912356Disabling compacts and flushes for region at 1732293912356Disabling writes for close at 1732293912356Writing region close event to WAL at 1732293912360 (+4 ms)Running coprocessor post-close hooks at 1732293912365 (+5 ms)Closed at 1732293912365 2024-11-22T16:45:12,365 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a. 2024-11-22T16:45:12,366 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4618c6346924,34043,1732293911743 already deleted, retry=false 2024-11-22T16:45:12,366 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4618c6346924,34043,1732293911743 expired; onlineServers=1 2024-11-22T16:45:12,368 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d45c41eb11994bda88dd600bd57fc6ea 2024-11-22T16:45:12,382 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/info/d45c41eb11994bda88dd600bd57fc6ea is 209, key is testBlockEvictionOnGracefulStop1732293909544,,1732293909548.ff67d88bc2134e831f87243685e5003a./info:regioninfo/1732293911797/Put/seqid=0 2024-11-22T16:45:12,383 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/info/d45c41eb11994bda88dd600bd57fc6ea 2024-11-22T16:45:12,388 INFO [regionserver/4618c6346924:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:12,391 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1665ce110a604a17b42dd0fe6cf9ec84 2024-11-22T16:45:12,400 INFO [regionserver/4618c6346924:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:12,405 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/ns/1665ce110a604a17b42dd0fe6cf9ec84 is 43, key is default/ns:d/1732293909421/Put/seqid=0 2024-11-22T16:45:12,405 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/ns/1665ce110a604a17b42dd0fe6cf9ec84 2024-11-22T16:45:12,414 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d7e8da8a90e94ec6bf1be78f31745815 2024-11-22T16:45:12,428 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/table/d7e8da8a90e94ec6bf1be78f31745815 is 80, key is testBlockEvictionOnGracefulStop1732293909544/table:state/1732293909925/Put/seqid=0 2024-11-22T16:45:12,429 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/table/d7e8da8a90e94ec6bf1be78f31745815 2024-11-22T16:45:12,431 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/info/d45c41eb11994bda88dd600bd57fc6ea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/d45c41eb11994bda88dd600bd57fc6ea 2024-11-22T16:45:12,432 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/d45c41eb11994bda88dd600bd57fc6ea, delay=970 ms 2024-11-22T16:45:12,433 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/d45c41eb11994bda88dd600bd57fc6ea, entries=14, sequenceid=13, filesize=7.5 K 2024-11-22T16:45:12,434 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/ns/1665ce110a604a17b42dd0fe6cf9ec84 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/1665ce110a604a17b42dd0fe6cf9ec84 2024-11-22T16:45:12,435 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/1665ce110a604a17b42dd0fe6cf9ec84, delay=957 ms 2024-11-22T16:45:12,436 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/1665ce110a604a17b42dd0fe6cf9ec84, entries=2, sequenceid=13, filesize=5.0 K 2024-11-22T16:45:12,436 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/.tmp/table/d7e8da8a90e94ec6bf1be78f31745815 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/d7e8da8a90e94ec6bf1be78f31745815 2024-11-22T16:45:12,438 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/d7e8da8a90e94ec6bf1be78f31745815, delay=917 ms 2024-11-22T16:45:12,438 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/d7e8da8a90e94ec6bf1be78f31745815, entries=2, sequenceid=13, filesize=5.3 K 2024-11-22T16:45:12,439 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=13, compaction requested=false 2024-11-22T16:45:12,440 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/info/d45c41eb11994bda88dd600bd57fc6ea 2024-11-22T16:45:12,441 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/ns/1665ce110a604a17b42dd0fe6cf9ec84 2024-11-22T16:45:12,442 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/table/d7e8da8a90e94ec6bf1be78f31745815 2024-11-22T16:45:12,447 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-22T16:45:12,447 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T16:45:12,447 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:45:12,448 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732293912356Running coprocessor pre-close hooks at 1732293912356Disabling compacts and flushes for region at 1732293912356Disabling writes for close at 1732293912357 (+1 ms)Obtaining lock to block concurrent updates at 1732293912357Preparing flush snapshotting stores in 1588230740 at 1732293912357Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732293912357Flushing stores of hbase:meta,,1.1588230740 at 1732293912357Flushing 1588230740/info: creating writer at 1732293912357Flushing 1588230740/info: appending metadata at 1732293912381 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732293912382 (+1 ms)Flushing 1588230740/ns: creating writer at 1732293912385 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732293912404 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732293912404Flushing 1588230740/table: creating writer at 1732293912407 (+3 ms)Flushing 1588230740/table: appending metadata at 1732293912427 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732293912427Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@838c7f7: reopening flushed file at 1732293912431 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d0444a4: reopening flushed file at 1732293912434 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3791cc90: reopening flushed file at 1732293912436 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=13, compaction requested=false at 1732293912439 (+3 ms)Writing region close event to WAL at 1732293912442 (+3 ms)Running coprocessor post-close hooks at 1732293912447 (+5 ms)Closed at 1732293912447 2024-11-22T16:45:12,448 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T16:45:12,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:12,464 INFO [RS:2;4618c6346924:34043 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:45:12,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34043-0x1014091424f0003, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:12,465 INFO [RS:2;4618c6346924:34043 {}] regionserver.HRegionServer(1031): Exiting; stopping=4618c6346924,34043,1732293911743; zookeeper connection closed. 2024-11-22T16:45:12,465 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3176a2bb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3176a2bb 2024-11-22T16:45:12,557 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(976): stopping server 4618c6346924,40897,1732293908512; all regions closed. 2024-11-22T16:45:12,559 DEBUG [RS:1;4618c6346924:40897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs 2024-11-22T16:45:12,559 INFO [RS:1;4618c6346924:40897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C40897%2C1732293908512.meta:.meta(num 1732293909371) 2024-11-22T16:45:12,561 DEBUG [RS:1;4618c6346924:40897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/oldWALs 2024-11-22T16:45:12,561 INFO [RS:1;4618c6346924:40897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C40897%2C1732293908512:(num 1732293909025) 2024-11-22T16:45:12,561 DEBUG [RS:1;4618c6346924:40897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:45:12,561 INFO [RS:1;4618c6346924:40897 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:45:12,561 INFO [RS:1;4618c6346924:40897 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:45:12,561 INFO [RS:1;4618c6346924:40897 {}] hbase.ChoreService(370): Chore service for: regionserver/4618c6346924:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T16:45:12,561 INFO [RS:1;4618c6346924:40897 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:45:12,561 INFO [regionserver/4618c6346924:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:45:12,561 INFO [RS:1;4618c6346924:40897 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40897 2024-11-22T16:45:12,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4618c6346924,40897,1732293908512 2024-11-22T16:45:12,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:45:12,564 INFO [RS:1;4618c6346924:40897 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:45:12,565 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4618c6346924,40897,1732293908512] 2024-11-22T16:45:12,568 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4618c6346924,40897,1732293908512 already deleted, retry=false 2024-11-22T16:45:12,568 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4618c6346924,40897,1732293908512 expired; onlineServers=0 2024-11-22T16:45:12,568 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4618c6346924,33939,1732293908431' ***** 2024-11-22T16:45:12,568 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T16:45:12,568 INFO [M:0;4618c6346924:33939 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:45:12,568 INFO [M:0;4618c6346924:33939 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:45:12,568 DEBUG [M:0;4618c6346924:33939 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T16:45:12,568 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T16:45:12,568 DEBUG [M:0;4618c6346924:33939 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T16:45:12,568 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293908755 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293908755,5,FailOnTimeoutGroup] 2024-11-22T16:45:12,568 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293908756 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293908756,5,FailOnTimeoutGroup] 2024-11-22T16:45:12,568 INFO [M:0;4618c6346924:33939 {}] hbase.ChoreService(370): Chore service for: master/4618c6346924:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T16:45:12,569 INFO [M:0;4618c6346924:33939 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:45:12,569 DEBUG [M:0;4618c6346924:33939 {}] master.HMaster(1795): Stopping service threads 2024-11-22T16:45:12,569 INFO [M:0;4618c6346924:33939 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T16:45:12,569 INFO [M:0;4618c6346924:33939 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:45:12,569 INFO [M:0;4618c6346924:33939 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:45:12,569 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T16:45:12,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T16:45:12,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:45:12,570 DEBUG [M:0;4618c6346924:33939 {}] zookeeper.ZKUtil(347): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T16:45:12,570 WARN [M:0;4618c6346924:33939 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T16:45:12,570 INFO [M:0;4618c6346924:33939 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/.lastflushedseqids 2024-11-22T16:45:12,578 INFO [M:0;4618c6346924:33939 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:45:12,578 INFO [M:0;4618c6346924:33939 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T16:45:12,578 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:45:12,578 INFO [M:0;4618c6346924:33939 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:12,578 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:12,578 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:45:12,578 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:12,578 INFO [M:0;4618c6346924:33939 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-22T16:45:12,588 DEBUG [M:0;4618c6346924:33939 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e05cd6cd024e448c9346f0cc30fd84b4 2024-11-22T16:45:12,602 DEBUG [M:0;4618c6346924:33939 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e05cd6cd024e448c9346f0cc30fd84b4 is 82, key is hbase:meta,,1/info:regioninfo/1732293909407/Put/seqid=0 2024-11-22T16:45:12,602 INFO [M:0;4618c6346924:33939 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e05cd6cd024e448c9346f0cc30fd84b4 2024-11-22T16:45:12,610 DEBUG [M:0;4618c6346924:33939 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 65fc27a618194363b3772da2a0ec9271 2024-11-22T16:45:12,625 DEBUG [M:0;4618c6346924:33939 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/65fc27a618194363b3772da2a0ec9271 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732293909929/Put/seqid=0 2024-11-22T16:45:12,625 INFO [M:0;4618c6346924:33939 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/65fc27a618194363b3772da2a0ec9271 2024-11-22T16:45:12,634 DEBUG [M:0;4618c6346924:33939 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c83846fb93da4eabbcdf604ece298276 2024-11-22T16:45:12,648 DEBUG [M:0;4618c6346924:33939 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c83846fb93da4eabbcdf604ece298276 is 69, key is 4618c6346924,34043,1732293911743/rs:state/1732293911780/Put/seqid=0 2024-11-22T16:45:12,649 INFO [M:0;4618c6346924:33939 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c83846fb93da4eabbcdf604ece298276 2024-11-22T16:45:12,651 INFO [M:0;4618c6346924:33939 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c83846fb93da4eabbcdf604ece298276 2024-11-22T16:45:12,651 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e05cd6cd024e448c9346f0cc30fd84b4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e05cd6cd024e448c9346f0cc30fd84b4 2024-11-22T16:45:12,653 INFO [M:0;4618c6346924:33939 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e05cd6cd024e448c9346f0cc30fd84b4, entries=8, sequenceid=83, filesize=5.5 K 2024-11-22T16:45:12,654 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/65fc27a618194363b3772da2a0ec9271 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/65fc27a618194363b3772da2a0ec9271 2024-11-22T16:45:12,655 INFO [M:0;4618c6346924:33939 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/65fc27a618194363b3772da2a0ec9271, entries=9, sequenceid=83, filesize=6.4 K 2024-11-22T16:45:12,656 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c83846fb93da4eabbcdf604ece298276 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c83846fb93da4eabbcdf604ece298276 2024-11-22T16:45:12,657 INFO [M:0;4618c6346924:33939 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c83846fb93da4eabbcdf604ece298276 2024-11-22T16:45:12,657 INFO [M:0;4618c6346924:33939 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18e47655-bd55-88fc-77ab-869d5dda2844/1ab50dda-8b95-cfab-3899-343e437f4bd3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c83846fb93da4eabbcdf604ece298276, entries=3, sequenceid=83, filesize=5.3 K 2024-11-22T16:45:12,658 INFO [M:0;4618c6346924:33939 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=83, compaction requested=false 2024-11-22T16:45:12,660 INFO [M:0;4618c6346924:33939 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:45:12,660 DEBUG [M:0;4618c6346924:33939 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732293912578Disabling compacts and flushes for region at 1732293912578Disabling writes for close at 1732293912578Obtaining lock to block concurrent updates at 1732293912578Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732293912578Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732293912579 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732293912579Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732293912579Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732293912601 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732293912601Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732293912604 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732293912624 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732293912624Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732293912627 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732293912648 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732293912648Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71332250: reopening flushed file at 1732293912651 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31c4d60e: reopening flushed file at 1732293912653 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@706496de: reopening flushed file at 1732293912655 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=83, compaction requested=false at 1732293912658 (+3 ms)Writing region close event to WAL at 1732293912660 (+2 ms)Closed at 1732293912660 2024-11-22T16:45:12,661 INFO [M:0;4618c6346924:33939 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T16:45:12,661 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:45:12,661 INFO [M:0;4618c6346924:33939 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33939 2024-11-22T16:45:12,661 INFO [M:0;4618c6346924:33939 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:45:12,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:12,665 INFO [RS:1;4618c6346924:40897 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:45:12,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40897-0x1014091424f0002, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:12,665 INFO [RS:1;4618c6346924:40897 {}] regionserver.HRegionServer(1031): Exiting; stopping=4618c6346924,40897,1732293908512; zookeeper connection closed. 2024-11-22T16:45:12,666 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4cb18c97 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4cb18c97 2024-11-22T16:45:12,666 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-22T16:45:12,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:12,764 INFO [M:0;4618c6346924:33939 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:45:12,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33939-0x1014091424f0000, quorum=127.0.0.1:62573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:45:12,767 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T16:45:12,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T16:45:12,769 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:45:12,775 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=223 (was 243), ProcessCount=11 (was 11), AvailableMemoryMB=7631 (was 7716)