2024-11-14 04:49:49,401 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec 2024-11-14 04:49:49,418 main DEBUG Took 0.015022 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 04:49:49,425 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 04:49:49,430 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 04:49:49,431 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 04:49:49,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,452 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 04:49:49,493 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,502 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,503 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,505 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,506 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,508 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,508 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,509 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,510 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,510 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,511 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,512 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,513 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,514 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:49:49,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,518 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 04:49:49,520 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:49:49,522 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 04:49:49,524 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 04:49:49,525 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 04:49:49,526 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 04:49:49,527 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 04:49:49,539 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 04:49:49,542 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 04:49:49,545 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 04:49:49,545 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 04:49:49,546 main DEBUG createAppenders(={Console}) 2024-11-14 04:49:49,547 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec initialized 2024-11-14 04:49:49,547 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec 2024-11-14 04:49:49,547 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec OK. 2024-11-14 04:49:49,548 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 04:49:49,548 main DEBUG OutputStream closed 2024-11-14 04:49:49,549 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 04:49:49,549 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 04:49:49,550 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@bae7dc0 OK 2024-11-14 04:49:49,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 04:49:49,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 04:49:49,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 04:49:49,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 04:49:49,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 04:49:49,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 04:49:49,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 04:49:49,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 04:49:49,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 04:49:49,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 04:49:49,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 04:49:49,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 04:49:49,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 04:49:49,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 04:49:49,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 04:49:49,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 04:49:49,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 04:49:49,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 04:49:49,695 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 04:49:49,695 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@2b76ff4e) with optional ClassLoader: null 2024-11-14 04:49:49,695 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 04:49:49,696 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@2b76ff4e] started OK. 2024-11-14T04:49:49,719 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-14 04:49:49,726 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 04:49:49,727 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T04:49:50,258 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0 2024-11-14T04:49:50,312 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=761, ProcessCount=11, AvailableMemoryMB=7684 2024-11-14T04:49:50,449 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T04:49:50,596 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:50,630 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/cluster_e1a2e066-e9f5-c9c4-e3a2-6b700629b24c, deleteOnExit=true 2024-11-14T04:49:50,795 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/cluster_e1a2e066-e9f5-c9c4-e3a2-6b700629b24c/zookeeper_0, clientPort=61305, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/cluster_e1a2e066-e9f5-c9c4-e3a2-6b700629b24c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/cluster_e1a2e066-e9f5-c9c4-e3a2-6b700629b24c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T04:49:50,819 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61305 2024-11-14T04:49:50,821 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:50,950 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 with version=8 2024-11-14T04:49:50,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/hbase-staging 2024-11-14T04:49:51,110 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T04:49:51,503 INFO [Time-limited test {}] client.ConnectionUtils(128): master/44289c305c03:0 server-side Connection retries=45 2024-11-14T04:49:51,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:51,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:51,532 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:49:51,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:51,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:49:51,725 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T04:49:51,830 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T04:49:51,893 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T04:49:51,909 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T04:49:51,918 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:49:51,985 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21225 (auto-detected) 2024-11-14T04:49:51,987 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T04:49:52,024 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41663 2024-11-14T04:49:52,069 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41663 connecting to ZooKeeper ensemble=127.0.0.1:61305 2024-11-14T04:49:52,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416630x0, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:49:52,131 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41663-0x100ef948d320000 connected 2024-11-14T04:49:52,173 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:52,175 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:52,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:52,189 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643, hbase.cluster.distributed=false 2024-11-14T04:49:52,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:49:52,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41663 2024-11-14T04:49:52,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41663 2024-11-14T04:49:52,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41663 2024-11-14T04:49:52,236 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41663 2024-11-14T04:49:52,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41663 2024-11-14T04:49:52,361 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/44289c305c03:0 server-side Connection retries=45 2024-11-14T04:49:52,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:52,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:52,365 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:49:52,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:52,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:49:52,369 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:49:52,373 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:49:52,374 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45709 2024-11-14T04:49:52,377 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45709 connecting to ZooKeeper ensemble=127.0.0.1:61305 2024-11-14T04:49:52,378 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:52,382 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:52,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457090x0, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:49:52,391 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45709-0x100ef948d320001 connected 2024-11-14T04:49:52,394 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:52,402 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:49:52,430 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.cache 2024-11-14T04:49:52,438 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T04:49:52,439 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T04:49:52,441 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T04:49:52,462 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T04:49:52,468 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T04:49:52,470 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T04:49:52,484 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:49:52,490 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T04:49:52,490 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.persistence is complete. 2024-11-14T04:49:52,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T04:49:52,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:49:52,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45709 2024-11-14T04:49:52,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45709 2024-11-14T04:49:52,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45709 2024-11-14T04:49:52,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45709 2024-11-14T04:49:52,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45709 2024-11-14T04:49:52,554 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/44289c305c03:0 server-side Connection retries=45 2024-11-14T04:49:52,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:52,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:52,555 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:49:52,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:52,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:49:52,556 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:49:52,557 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:49:52,562 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42037 2024-11-14T04:49:52,565 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42037 connecting to ZooKeeper ensemble=127.0.0.1:61305 2024-11-14T04:49:52,566 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:52,567 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:52,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420370x0, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:49:52,575 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:420370x0, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:52,576 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:49:52,579 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42037-0x100ef948d320002 connected 2024-11-14T04:49:52,580 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.cache 2024-11-14T04:49:52,580 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T04:49:52,581 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T04:49:52,581 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T04:49:52,582 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T04:49:52,583 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T04:49:52,585 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T04:49:52,585 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.persistence is complete. 2024-11-14T04:49:52,589 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T04:49:52,592 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:49:52,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T04:49:52,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:49:52,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42037 2024-11-14T04:49:52,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42037 2024-11-14T04:49:52,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42037 2024-11-14T04:49:52,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42037 2024-11-14T04:49:52,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42037 2024-11-14T04:49:52,725 DEBUG [M:0;44289c305c03:41663 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;44289c305c03:41663 2024-11-14T04:49:52,727 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/44289c305c03,41663,1731559791235 2024-11-14T04:49:52,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:52,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:52,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:52,744 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/44289c305c03,41663,1731559791235 2024-11-14T04:49:52,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T04:49:52,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T04:49:52,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:52,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:52,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:52,792 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T04:49:52,794 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/44289c305c03,41663,1731559791235 from backup master directory 2024-11-14T04:49:52,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/44289c305c03,41663,1731559791235 2024-11-14T04:49:52,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:52,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:52,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:52,807 WARN [master/44289c305c03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:49:52,808 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=44289c305c03,41663,1731559791235 2024-11-14T04:49:52,812 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T04:49:52,814 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T04:49:52,833 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/hbase.id] with ID: 9d76caec-f6c8-434a-baeb-d24b52869c56 2024-11-14T04:49:52,837 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.tmp/hbase.id 2024-11-14T04:49:52,855 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/hbase.id] 2024-11-14T04:49:53,002 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:53,009 INFO [master/44289c305c03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T04:49:53,037 INFO [master/44289c305c03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-14T04:49:53,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:53,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:53,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:53,094 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T04:49:53,099 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T04:49:53,195 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:49:53,206 INFO [master/44289c305c03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:49:53,429 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store 2024-11-14T04:49:53,498 INFO [master/44289c305c03:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T04:49:53,504 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:53,506 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T04:49:53,507 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:53,507 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:53,509 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T04:49:53,509 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:53,510 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:53,511 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731559793506Disabling compacts and flushes for region at 1731559793506Disabling writes for close at 1731559793509 (+3 ms)Writing region close event to WAL at 1731559793510 (+1 ms)Closed at 1731559793510 2024-11-14T04:49:53,512 WARN [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/.initializing 2024-11-14T04:49:53,525 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/WALs/44289c305c03,41663,1731559791235 2024-11-14T04:49:53,580 INFO [master/44289c305c03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:49:53,631 INFO [master/44289c305c03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=44289c305c03%2C41663%2C1731559791235, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/WALs/44289c305c03,41663,1731559791235, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/oldWALs, maxLogs=10 2024-11-14T04:49:53,704 INFO [master/44289c305c03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/WALs/44289c305c03,41663,1731559791235/44289c305c03%2C41663%2C1731559791235.1731559793638 2024-11-14T04:49:53,706 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:49:53,707 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:49:53,708 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:53,714 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,715 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T04:49:53,858 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:53,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:53,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T04:49:53,879 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:53,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:53,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T04:49:53,926 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:53,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:53,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T04:49:53,944 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:53,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:53,945 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,950 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,951 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,955 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,956 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,962 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:49:53,965 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:53,987 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:49:53,988 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61752728, jitterRate=-0.07981264591217041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:49:53,997 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731559793738Initializing all the Stores at 1731559793740 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559793740Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559793744 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559793744Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559793744Cleaning up temporary data from old regions at 1731559793956 (+212 ms)Region opened successfully at 1731559793997 (+41 ms) 2024-11-14T04:49:53,999 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T04:49:54,046 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f650629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=44289c305c03/172.17.0.2:0 2024-11-14T04:49:54,094 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T04:49:54,111 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T04:49:54,112 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T04:49:54,116 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T04:49:54,117 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T04:49:54,131 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-14T04:49:54,132 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T04:49:54,168 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T04:49:54,183 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T04:49:54,189 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T04:49:54,193 INFO [master/44289c305c03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T04:49:54,205 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T04:49:54,211 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T04:49:54,215 INFO [master/44289c305c03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T04:49:54,238 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T04:49:54,245 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T04:49:54,253 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T04:49:54,256 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T04:49:54,309 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T04:49:54,313 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T04:49:54,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:54,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:54,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:54,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,354 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=44289c305c03,41663,1731559791235, sessionid=0x100ef948d320000, setting cluster-up flag (Was=false) 2024-11-14T04:49:54,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,396 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T04:49:54,399 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=44289c305c03,41663,1731559791235 2024-11-14T04:49:54,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:54,417 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T04:49:54,420 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=44289c305c03,41663,1731559791235 2024-11-14T04:49:54,440 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T04:49:54,522 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(746): ClusterId : 9d76caec-f6c8-434a-baeb-d24b52869c56 2024-11-14T04:49:54,523 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(746): ClusterId : 9d76caec-f6c8-434a-baeb-d24b52869c56 2024-11-14T04:49:54,526 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:49:54,526 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:49:54,541 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:49:54,541 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:49:54,541 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:49:54,541 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:49:54,546 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:49:54,546 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:49:54,547 DEBUG [RS:1;44289c305c03:42037 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d2716f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=44289c305c03/172.17.0.2:0 2024-11-14T04:49:54,547 DEBUG [RS:0;44289c305c03:45709 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380cc2e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=44289c305c03/172.17.0.2:0 2024-11-14T04:49:54,578 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;44289c305c03:42037 2024-11-14T04:49:54,578 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;44289c305c03:45709 2024-11-14T04:49:54,583 INFO [RS:0;44289c305c03:45709 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:49:54,583 INFO [RS:1;44289c305c03:42037 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:49:54,583 INFO [RS:0;44289c305c03:45709 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:49:54,583 INFO [RS:1;44289c305c03:42037 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:49:54,584 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:49:54,583 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:49:54,590 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(2659): reportForDuty to master=44289c305c03,41663,1731559791235 with port=45709, startcode=1731559792304 2024-11-14T04:49:54,590 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T04:49:54,591 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(2659): reportForDuty to master=44289c305c03,41663,1731559791235 with port=42037, startcode=1731559792553 2024-11-14T04:49:54,607 INFO [master/44289c305c03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T04:49:54,608 DEBUG [RS:0;44289c305c03:45709 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:49:54,609 DEBUG [RS:1;44289c305c03:42037 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:49:54,621 INFO [master/44289c305c03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T04:49:54,628 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 44289c305c03,41663,1731559791235 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T04:49:54,672 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:54,672 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:54,672 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:54,673 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:54,673 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/44289c305c03:0, corePoolSize=10, maxPoolSize=10 2024-11-14T04:49:54,680 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:54,681 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/44289c305c03:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:49:54,681 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:54,726 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731559824726 2024-11-14T04:49:54,728 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T04:49:54,729 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T04:49:54,731 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:49:54,731 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T04:49:54,735 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T04:49:54,736 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T04:49:54,736 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T04:49:54,737 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T04:49:54,745 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42271, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:49:54,742 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:54,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50857, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:49:54,752 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:54,753 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T04:49:54,753 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T04:49:54,754 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T04:49:54,755 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T04:49:54,757 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41663 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T04:49:54,767 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41663 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T04:49:54,772 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T04:49:54,772 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T04:49:54,784 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.large.0-1731559794778,5,FailOnTimeoutGroup] 2024-11-14T04:49:54,793 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.small.0-1731559794785,5,FailOnTimeoutGroup] 2024-11-14T04:49:54,794 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:54,794 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T04:49:54,795 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:54,796 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:54,824 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T04:49:54,824 WARN [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T04:49:54,829 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T04:49:54,829 WARN [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T04:49:54,863 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T04:49:54,863 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:49:54,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:54,926 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(2659): reportForDuty to master=44289c305c03,41663,1731559791235 with port=45709, startcode=1731559792304 2024-11-14T04:49:54,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T04:49:54,929 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 44289c305c03,45709,1731559792304 2024-11-14T04:49:54,931 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(2659): reportForDuty to master=44289c305c03,41663,1731559791235 with port=42037, startcode=1731559792553 2024-11-14T04:49:54,932 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41663 {}] master.ServerManager(517): Registering regionserver=44289c305c03,45709,1731559792304 2024-11-14T04:49:54,946 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:49:54,947 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T04:49:54,947 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:49:54,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:49:54,955 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T04:49:54,955 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:54,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:54,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T04:49:54,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 44289c305c03,42037,1731559792553 2024-11-14T04:49:54,957 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41663 {}] master.ServerManager(517): Registering regionserver=44289c305c03,42037,1731559792553 2024-11-14T04:49:54,962 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:49:54,962 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T04:49:54,962 DEBUG [RS:0;44289c305c03:45709 {}] zookeeper.ZKUtil(111): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/44289c305c03,45709,1731559792304 2024-11-14T04:49:54,962 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:49:54,963 WARN [RS:0;44289c305c03:45709 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:49:54,963 INFO [RS:0;44289c305c03:45709 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:49:54,963 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,45709,1731559792304 2024-11-14T04:49:54,970 DEBUG [RS:1;44289c305c03:42037 {}] zookeeper.ZKUtil(111): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/44289c305c03,42037,1731559792553 2024-11-14T04:49:54,971 WARN [RS:1;44289c305c03:42037 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:49:54,971 INFO [RS:1;44289c305c03:42037 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:49:54,971 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,42037,1731559792553 2024-11-14T04:49:54,976 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [44289c305c03,45709,1731559792304] 2024-11-14T04:49:54,977 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [44289c305c03,42037,1731559792553] 2024-11-14T04:49:54,999 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T04:49:54,999 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:54,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:55,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T04:49:55,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T04:49:55,026 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:55,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:55,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T04:49:55,042 INFO [RS:1;44289c305c03:42037 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:49:55,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T04:49:55,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:55,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:55,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T04:49:55,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:49:55,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:49:55,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T04:49:55,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T04:49:55,058 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:49:55,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T04:49:55,059 INFO [RS:0;44289c305c03:45709 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:49:55,075 INFO [RS:0;44289c305c03:45709 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:49:55,075 INFO [RS:1;44289c305c03:42037 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:49:55,090 INFO [RS:1;44289c305c03:42037 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:49:55,090 INFO [RS:0;44289c305c03:45709 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:49:55,090 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,090 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,106 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:49:55,107 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68481915, jitterRate=0.020460054278373718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:49:55,107 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:49:55,111 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:49:55,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731559794918Initializing all the Stores at 1731559794918Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559794918Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559794928 (+10 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559794928Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559794928Cleaning up temporary data from old regions at 1731559795057 (+129 ms)Region opened successfully at 1731559795111 (+54 ms) 2024-11-14T04:49:55,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T04:49:55,112 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T04:49:55,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T04:49:55,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T04:49:55,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T04:49:55,122 INFO [RS:0;44289c305c03:45709 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:49:55,124 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,125 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,125 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,125 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T04:49:55,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731559795112Disabling compacts and flushes for region at 1731559795112Disabling writes for close at 1731559795112Writing region close event to WAL at 1731559795125 (+13 ms)Closed at 1731559795125 2024-11-14T04:49:55,126 INFO [RS:1;44289c305c03:42037 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:49:55,126 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,126 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,126 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,127 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,127 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,127 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,127 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/44289c305c03:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:49:55,128 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:49:55,128 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,128 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T04:49:55,128 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,128 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,128 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,129 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,129 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,130 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:49:55,129 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,130 DEBUG [RS:1;44289c305c03:42037 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:49:55,130 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,131 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,131 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/44289c305c03:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:49:55,131 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,131 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,132 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,132 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,132 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,133 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:55,134 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:49:55,134 DEBUG [RS:0;44289c305c03:45709 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:49:55,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T04:49:55,153 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T04:49:55,156 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T04:49:55,161 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,161 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,161 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,161 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,161 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,162 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,42037,1731559792553-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:49:55,165 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,165 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,166 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,166 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,166 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,166 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,45709,1731559792304-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:49:55,200 INFO [RS:1;44289c305c03:42037 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:49:55,203 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,42037,1731559792553-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,204 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,204 INFO [RS:1;44289c305c03:42037 {}] regionserver.Replication(171): 44289c305c03,42037,1731559792553 started 2024-11-14T04:49:55,230 INFO [RS:0;44289c305c03:45709 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:49:55,231 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,45709,1731559792304-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,231 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,231 INFO [RS:0;44289c305c03:45709 {}] regionserver.Replication(171): 44289c305c03,45709,1731559792304 started 2024-11-14T04:49:55,254 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,254 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:55,254 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1482): Serving as 44289c305c03,45709,1731559792304, RpcServer on 44289c305c03/172.17.0.2:45709, sessionid=0x100ef948d320001 2024-11-14T04:49:55,254 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1482): Serving as 44289c305c03,42037,1731559792553, RpcServer on 44289c305c03/172.17.0.2:42037, sessionid=0x100ef948d320002 2024-11-14T04:49:55,256 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:49:55,256 DEBUG [RS:1;44289c305c03:42037 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 44289c305c03,42037,1731559792553 2024-11-14T04:49:55,256 DEBUG [RS:1;44289c305c03:42037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,42037,1731559792553' 2024-11-14T04:49:55,257 DEBUG [RS:1;44289c305c03:42037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:49:55,257 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:49:55,258 DEBUG [RS:1;44289c305c03:42037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:49:55,259 DEBUG [RS:0;44289c305c03:45709 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 44289c305c03,45709,1731559792304 2024-11-14T04:49:55,259 DEBUG [RS:0;44289c305c03:45709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,45709,1731559792304' 2024-11-14T04:49:55,260 DEBUG [RS:0;44289c305c03:45709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:49:55,260 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:49:55,260 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:49:55,260 DEBUG [RS:1;44289c305c03:42037 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 44289c305c03,42037,1731559792553 2024-11-14T04:49:55,260 DEBUG [RS:1;44289c305c03:42037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,42037,1731559792553' 2024-11-14T04:49:55,260 DEBUG [RS:1;44289c305c03:42037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:49:55,261 DEBUG [RS:0;44289c305c03:45709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:49:55,262 DEBUG [RS:1;44289c305c03:42037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:49:55,262 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:49:55,262 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:49:55,262 DEBUG [RS:0;44289c305c03:45709 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 44289c305c03,45709,1731559792304 2024-11-14T04:49:55,262 DEBUG [RS:0;44289c305c03:45709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,45709,1731559792304' 2024-11-14T04:49:55,262 DEBUG [RS:0;44289c305c03:45709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:49:55,266 DEBUG [RS:0;44289c305c03:45709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:49:55,267 DEBUG [RS:1;44289c305c03:42037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:49:55,267 DEBUG [RS:0;44289c305c03:45709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:49:55,267 INFO [RS:1;44289c305c03:42037 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:49:55,267 INFO [RS:0;44289c305c03:45709 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:49:55,267 INFO [RS:1;44289c305c03:42037 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:49:55,267 INFO [RS:0;44289c305c03:45709 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:49:55,307 WARN [44289c305c03:41663 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T04:49:55,373 INFO [RS:1;44289c305c03:42037 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:49:55,373 INFO [RS:0;44289c305c03:45709 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:49:55,375 INFO [RS:0;44289c305c03:45709 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=44289c305c03%2C45709%2C1731559792304, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,45709,1731559792304, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs, maxLogs=55 2024-11-14T04:49:55,385 INFO [RS:1;44289c305c03:42037 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=44289c305c03%2C42037%2C1731559792553, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,42037,1731559792553, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs, maxLogs=55 2024-11-14T04:49:55,406 INFO [RS:0;44289c305c03:45709 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,45709,1731559792304/44289c305c03%2C45709%2C1731559792304.1731559795379 2024-11-14T04:49:55,412 DEBUG [RS:0;44289c305c03:45709 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:49:55,421 INFO [RS:1;44289c305c03:42037 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,42037,1731559792553/44289c305c03%2C42037%2C1731559792553.1731559795388 2024-11-14T04:49:55,422 DEBUG [RS:1;44289c305c03:42037 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:49:55,560 DEBUG [44289c305c03:41663 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T04:49:55,576 DEBUG [44289c305c03:41663 {}] balancer.BalancerClusterState(204): Hosts are {44289c305c03=0} racks are {/default-rack=0} 2024-11-14T04:49:55,581 DEBUG [44289c305c03:41663 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T04:49:55,581 DEBUG [44289c305c03:41663 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T04:49:55,581 DEBUG [44289c305c03:41663 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T04:49:55,582 DEBUG [44289c305c03:41663 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T04:49:55,582 INFO [44289c305c03:41663 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T04:49:55,582 INFO [44289c305c03:41663 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T04:49:55,582 DEBUG [44289c305c03:41663 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T04:49:55,592 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=44289c305c03,45709,1731559792304 2024-11-14T04:49:55,599 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 44289c305c03,45709,1731559792304, state=OPENING 2024-11-14T04:49:55,679 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T04:49:55,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:55,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:55,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:55,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:49:55,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:49:55,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:49:55,693 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T04:49:55,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=44289c305c03,45709,1731559792304}] 2024-11-14T04:49:55,873 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:49:55,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36257, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:49:55,886 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T04:49:55,886 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:49:55,887 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T04:49:55,889 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=44289c305c03%2C45709%2C1731559792304.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,45709,1731559792304, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs, maxLogs=55 2024-11-14T04:49:55,920 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,45709,1731559792304/44289c305c03%2C45709%2C1731559792304.meta.1731559795892.meta 2024-11-14T04:49:55,921 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:49:55,922 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:49:55,924 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T04:49:55,928 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T04:49:55,934 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T04:49:55,939 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T04:49:55,940 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:55,940 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T04:49:55,941 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T04:49:55,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T04:49:55,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T04:49:55,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:55,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:55,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T04:49:55,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T04:49:55,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:55,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:55,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T04:49:55,947 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T04:49:55,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:55,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:55,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T04:49:55,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T04:49:55,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:55,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:55,949 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T04:49:55,951 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:49:55,952 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:49:55,952 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T04:49:55,953 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T04:49:55,953 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:49:55,955 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T04:49:55,956 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66427081, jitterRate=-0.010159358382225037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:49:55,956 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T04:49:55,958 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731559795941Writing region info on filesystem at 1731559795942 (+1 ms)Initializing all the Stores at 1731559795942Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559795942Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559795942Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559795943 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559795943Cleaning up temporary data from old regions at 1731559795953 (+10 ms)Running coprocessor post-open hooks at 1731559795956 (+3 ms)Region opened successfully at 1731559795958 (+2 ms) 2024-11-14T04:49:55,966 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731559795864 2024-11-14T04:49:55,979 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T04:49:55,980 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T04:49:55,981 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=44289c305c03,45709,1731559792304 2024-11-14T04:49:55,983 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 44289c305c03,45709,1731559792304, state=OPEN 2024-11-14T04:49:55,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:49:55,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:49:55,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:49:55,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:49:55,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:49:55,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:49:55,987 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=44289c305c03,45709,1731559792304 2024-11-14T04:49:55,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T04:49:55,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=44289c305c03,45709,1731559792304 in 292 msec 2024-11-14T04:49:55,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T04:49:55,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 856 msec 2024-11-14T04:49:55,996 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:49:55,997 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T04:49:56,031 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T04:49:56,032 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=44289c305c03,45709,1731559792304, seqNum=-1] 2024-11-14T04:49:56,061 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:49:56,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34075, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:49:56,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5980 sec 2024-11-14T04:49:56,086 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731559796086, completionTime=-1 2024-11-14T04:49:56,090 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-14T04:49:56,090 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T04:49:56,122 INFO [master/44289c305c03:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T04:49:56,122 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731559856122 2024-11-14T04:49:56,122 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731559916122 2024-11-14T04:49:56,123 INFO [master/44289c305c03:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-14T04:49:56,125 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T04:49:56,131 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,41663,1731559791235-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:56,132 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,41663,1731559791235-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:56,132 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,41663,1731559791235-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:56,134 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-44289c305c03:41663, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:56,134 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:56,135 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:56,142 DEBUG [master/44289c305c03:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T04:49:56,163 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.353sec 2024-11-14T04:49:56,164 INFO [master/44289c305c03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T04:49:56,165 INFO [master/44289c305c03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T04:49:56,166 INFO [master/44289c305c03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T04:49:56,167 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T04:49:56,167 INFO [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T04:49:56,168 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,41663,1731559791235-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:49:56,169 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,41663,1731559791235-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T04:49:56,171 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T04:49:56,172 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T04:49:56,172 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,41663,1731559791235-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:56,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e45a54f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:49:56,239 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T04:49:56,240 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T04:49:56,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 44289c305c03,41663,-1 for getting cluster id 2024-11-14T04:49:56,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T04:49:56,260 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9d76caec-f6c8-434a-baeb-d24b52869c56' 2024-11-14T04:49:56,263 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T04:49:56,263 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9d76caec-f6c8-434a-baeb-d24b52869c56" 2024-11-14T04:49:56,264 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d089b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:49:56,264 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [44289c305c03,41663,-1] 2024-11-14T04:49:56,268 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T04:49:56,270 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:56,272 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T04:49:56,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf3531f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:49:56,275 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T04:49:56,284 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=44289c305c03,45709,1731559792304, seqNum=-1] 2024-11-14T04:49:56,285 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:49:56,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:49:56,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=44289c305c03,41663,1731559791235 2024-11-14T04:49:56,322 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T04:49:56,327 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 44289c305c03,41663,1731559791235 2024-11-14T04:49:56,329 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@39a70c00 2024-11-14T04:49:56,330 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T04:49:56,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T04:49:56,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731559796317', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T04:49:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 2024-11-14T04:49:56,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T04:49:56,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731559796317" procId is: 4 2024-11-14T04:49:56,368 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:56,370 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T04:49:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:49:56,391 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2aca626354c0c6ec981bddd10eb8d87c, NAME => 'testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731559796317', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:49:56,399 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:56,399 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(1722): Closing 2aca626354c0c6ec981bddd10eb8d87c, disabling compactions & flushes 2024-11-14T04:49:56,400 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:56,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:56,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. after waiting 0 ms 2024-11-14T04:49:56,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:56,400 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:56,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731559796317-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2aca626354c0c6ec981bddd10eb8d87c: Waiting for close lock at 1731559796399Disabling compacts and flushes for region at 1731559796399Disabling writes for close at 1731559796400 (+1 ms)Writing region close event to WAL at 1731559796400Closed at 1731559796400 2024-11-14T04:49:56,401 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T04:49:56,406 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731559796402"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731559796402"}]},"ts":"1731559796402"} 2024-11-14T04:49:56,410 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T04:49:56,412 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T04:49:56,416 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731559796317","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731559796412"}]},"ts":"1731559796412"} 2024-11-14T04:49:56,420 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731559796317, state=ENABLING in hbase:meta 2024-11-14T04:49:56,420 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {44289c305c03=0} racks are {/default-rack=0} 2024-11-14T04:49:56,421 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T04:49:56,421 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T04:49:56,421 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T04:49:56,421 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T04:49:56,421 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T04:49:56,421 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T04:49:56,421 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T04:49:56,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, ASSIGN}] 2024-11-14T04:49:56,424 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, ASSIGN 2024-11-14T04:49:56,425 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, ASSIGN; state=OFFLINE, location=44289c305c03,45709,1731559792304; forceNewPlan=false, retain=false 2024-11-14T04:49:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:49:56,578 INFO [44289c305c03:41663 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T04:49:56,579 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2aca626354c0c6ec981bddd10eb8d87c, regionState=OPENING, regionLocation=44289c305c03,45709,1731559792304 2024-11-14T04:49:56,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, ASSIGN because future has completed 2024-11-14T04:49:56,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,45709,1731559792304}] 2024-11-14T04:49:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:49:56,741 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:56,741 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2aca626354c0c6ec981bddd10eb8d87c, NAME => 'testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:49:56,742 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731559796317 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,742 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:56,743 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,743 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,744 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,749 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aca626354c0c6ec981bddd10eb8d87c columnFamilyName family 2024-11-14T04:49:56,749 DEBUG [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:56,750 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.HStore(327): Store=2aca626354c0c6ec981bddd10eb8d87c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:56,750 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,751 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,752 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,752 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,752 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,753 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,761 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:49:56,761 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2aca626354c0c6ec981bddd10eb8d87c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63199865, jitterRate=-0.058248624205589294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:49:56,762 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:56,763 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2aca626354c0c6ec981bddd10eb8d87c: Running coprocessor pre-open hook at 1731559796743Writing region info on filesystem at 1731559796743Initializing all the Stores at 1731559796743Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559796743Cleaning up temporary data from old regions at 1731559796752 (+9 ms)Running coprocessor post-open hooks at 1731559796762 (+10 ms)Region opened successfully at 1731559796762 2024-11-14T04:49:56,764 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c., pid=6, masterSystemTime=1731559796737 2024-11-14T04:49:56,768 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:56,768 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:56,768 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2aca626354c0c6ec981bddd10eb8d87c, regionState=OPEN, openSeqNum=2, regionLocation=44289c305c03,45709,1731559792304 2024-11-14T04:49:56,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,45709,1731559792304 because future has completed 2024-11-14T04:49:56,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T04:49:56,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,45709,1731559792304 in 192 msec 2024-11-14T04:49:56,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T04:49:56,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, ASSIGN in 356 msec 2024-11-14T04:49:56,783 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T04:49:56,783 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731559796317","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731559796783"}]},"ts":"1731559796783"} 2024-11-14T04:49:56,786 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731559796317, state=ENABLED in hbase:meta 2024-11-14T04:49:56,787 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T04:49:56,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731559796317 in 430 msec 2024-11-14T04:49:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:49:57,011 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731559796317 get assigned. Timeout = 60000ms 2024-11-14T04:49:57,011 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731559796317 completed 2024-11-14T04:49:57,014 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T04:49:57,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731559796317 assigned to meta. Checking AM states. 2024-11-14T04:49:57,022 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T04:49:57,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731559796317 assigned. 2024-11-14T04:49:57,032 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731559796317', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c., hostname=44289c305c03,45709,1731559792304, seqNum=2] 2024-11-14T04:49:57,043 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2aca626354c0c6ec981bddd10eb8d87c 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-14T04:49:57,068 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b950417021e04676a46ed484b035c891 2024-11-14T04:49:57,106 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/.tmp/family/b950417021e04676a46ed484b035c891 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-14T04:49:57,112 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/.tmp/family/b950417021e04676a46ed484b035c891 2024-11-14T04:49:57,150 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/.tmp/family/b950417021e04676a46ed484b035c891 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891 2024-11-14T04:49:57,151 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891, delay=920 ms 2024-11-14T04:49:57,153 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891, entries=2, sequenceid=6, filesize=4.9 K 2024-11-14T04:49:57,159 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 2aca626354c0c6ec981bddd10eb8d87c in 115ms, sequenceid=6, compaction requested=false 2024-11-14T04:49:57,160 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731559796317' 2024-11-14T04:49:57,162 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2aca626354c0c6ec981bddd10eb8d87c: 2024-11-14T04:49:58,075 DEBUG [hfile-prefetch-1731559797152 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b950417021e04676a46ed484b035c891_0. 2024-11-14T04:49:58,080 DEBUG [hfile-prefetch-1731559797152 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b950417021e04676a46ed484b035c891_117. 2024-11-14T04:49:58,081 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891, with total blocks 2, and data blocks 1 2024-11-14T04:49:58,081 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T04:49:58,082 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T04:49:58,082 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891. Will sleep 100ms and try the verification again. 2024-11-14T04:49:58,109 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T04:49:58,109 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T04:49:58,109 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:49:58,110 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:49:58,133 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T04:49:58,133 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731559798109 2024-11-14T04:49:58,133 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T04:49:58,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=2aca626354c0c6ec981bddd10eb8d87c, source=44289c305c03,45709,1731559792304, destination=44289c305c03,42037,1731559792553, warming up region on 44289c305c03,42037,1731559792553 2024-11-14T04:49:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:49:58,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=2aca626354c0c6ec981bddd10eb8d87c, source=44289c305c03,45709,1731559792304, destination=44289c305c03,42037,1731559792553, running balancer 2024-11-14T04:49:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, REOPEN/MOVE 2024-11-14T04:49:58,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:49:58,180 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, REOPEN/MOVE 2024-11-14T04:49:58,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(7855): Warmup {ENCODED => 2aca626354c0c6ec981bddd10eb8d87c, NAME => 'testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:49:58,183 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891, with total blocks 2, and data blocks 1 2024-11-14T04:49:58,183 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-14T04:49:58,183 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2294): found block for file b950417021e04676a46ed484b035c891 in the backing map. Acquiring read lock for offset 0 2024-11-14T04:49:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:58,183 DEBUG [hfile-prefetch-1731559797152 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891 has now been fully cached. 2024-11-14T04:49:58,184 DEBUG [hfile-prefetch-1731559797152 {}] hfile.PrefetchExecutor(121): Prefetch completed for b950417021e04676a46ed484b035c891 2024-11-14T04:49:58,184 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2aca626354c0c6ec981bddd10eb8d87c, regionState=CLOSING, regionLocation=44289c305c03,45709,1731559792304 2024-11-14T04:49:58,184 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,185 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aca626354c0c6ec981bddd10eb8d87c columnFamilyName family 2024-11-14T04:49:58,186 DEBUG [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:58,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, REOPEN/MOVE because future has completed 2024-11-14T04:49:58,188 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-14T04:49:58,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,45709,1731559792304}] 2024-11-14T04:49:58,190 DEBUG [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891 2024-11-14T04:49:58,190 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.HStore(327): Store=2aca626354c0c6ec981bddd10eb8d87c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(1722): Closing 2aca626354c0c6ec981bddd10eb8d87c, disabling compactions & flushes 2024-11-14T04:49:58,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. after waiting 0 ms 2024-11-14T04:49:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] regionserver.HRegion(1676): Region close journal for 2aca626354c0c6ec981bddd10eb8d87c: Waiting for close lock at 1731559798191Disabling compacts and flushes for region at 1731559798191Disabling writes for close at 1731559798191Writing region close event to WAL at 1731559798193 (+2 ms)Closed at 1731559798193 2024-11-14T04:49:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41663 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-14T04:49:58,347 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,348 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-14T04:49:58,349 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 2aca626354c0c6ec981bddd10eb8d87c, disabling compactions & flushes 2024-11-14T04:49:58,349 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,349 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,349 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. after waiting 0 ms 2024-11-14T04:49:58,349 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,351 DEBUG [StoreFileCloser-2aca626354c0c6ec981bddd10eb8d87c-family-1 {}] bucket.BucketCache(769): Removing file b950417021e04676a46ed484b035c891 for region 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,352 DEBUG [StoreFileCloser-2aca626354c0c6ec981bddd10eb8d87c-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file b950417021e04676a46ed484b035c891, starting offset: 0, end offset: 9223372036854775807 2024-11-14T04:49:58,353 DEBUG [StoreFileCloser-2aca626354c0c6ec981bddd10eb8d87c-family-1 {}] bucket.BucketCache(857): removed key b950417021e04676a46ed484b035c891_0 from back map with offset lock 2094080 in the evict process 2024-11-14T04:49:58,362 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T04:49:58,364 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,364 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 2aca626354c0c6ec981bddd10eb8d87c: Waiting for close lock at 1731559798349Running coprocessor pre-close hooks at 1731559798349Disabling compacts and flushes for region at 1731559798349Disabling writes for close at 1731559798349Writing region close event to WAL at 1731559798354 (+5 ms)Running coprocessor post-close hooks at 1731559798362 (+8 ms)Closed at 1731559798364 (+2 ms) 2024-11-14T04:49:58,364 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 2aca626354c0c6ec981bddd10eb8d87c move to 44289c305c03,42037,1731559792553 record at close sequenceid=6 2024-11-14T04:49:58,368 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,368 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2aca626354c0c6ec981bddd10eb8d87c, regionState=CLOSED 2024-11-14T04:49:58,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,45709,1731559792304 because future has completed 2024-11-14T04:49:58,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T04:49:58,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,45709,1731559792304 in 184 msec 2024-11-14T04:49:58,375 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, REOPEN/MOVE; state=CLOSED, location=44289c305c03,42037,1731559792553; forceNewPlan=false, retain=false 2024-11-14T04:49:58,378 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T04:49:58,378 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T04:49:58,378 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:49:58,378 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:49:58,382 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T04:49:58,382 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731559798378 2024-11-14T04:49:58,382 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T04:49:58,526 INFO [44289c305c03:41663 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T04:49:58,526 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2aca626354c0c6ec981bddd10eb8d87c, regionState=OPENING, regionLocation=44289c305c03,42037,1731559792553 2024-11-14T04:49:58,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, REOPEN/MOVE because future has completed 2024-11-14T04:49:58,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,42037,1731559792553}] 2024-11-14T04:49:58,688 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,689 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 2aca626354c0c6ec981bddd10eb8d87c, NAME => 'testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:49:58,690 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731559796317 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,690 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:58,690 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,690 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,691 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,693 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aca626354c0c6ec981bddd10eb8d87c columnFamilyName family 2024-11-14T04:49:58,693 DEBUG [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:58,696 DEBUG [StoreFileOpener-2aca626354c0c6ec981bddd10eb8d87c-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891, delay=995 ms 2024-11-14T04:49:58,699 DEBUG [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891 2024-11-14T04:49:58,699 INFO [StoreOpener-2aca626354c0c6ec981bddd10eb8d87c-1 {}] regionserver.HStore(327): Store=2aca626354c0c6ec981bddd10eb8d87c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:58,699 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,700 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,701 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,701 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,701 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,708 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,709 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 2aca626354c0c6ec981bddd10eb8d87c; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69408365, jitterRate=0.03426523506641388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:49:58,709 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,710 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 2aca626354c0c6ec981bddd10eb8d87c: Running coprocessor pre-open hook at 1731559798690Writing region info on filesystem at 1731559798690Initializing all the Stores at 1731559798691 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559798691Cleaning up temporary data from old regions at 1731559798701 (+10 ms)Running coprocessor post-open hooks at 1731559798709 (+8 ms)Region opened successfully at 1731559798710 (+1 ms) 2024-11-14T04:49:58,711 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c., pid=9, masterSystemTime=1731559798684 2024-11-14T04:49:58,715 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,715 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,715 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2aca626354c0c6ec981bddd10eb8d87c, regionState=OPEN, openSeqNum=10, regionLocation=44289c305c03,42037,1731559792553 2024-11-14T04:49:58,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,42037,1731559792553 because future has completed 2024-11-14T04:49:58,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-14T04:49:58,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2aca626354c0c6ec981bddd10eb8d87c, server=44289c305c03,42037,1731559792553 in 190 msec 2024-11-14T04:49:58,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731559796317, region=2aca626354c0c6ec981bddd10eb8d87c, REOPEN/MOVE in 547 msec 2024-11-14T04:49:58,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T04:49:58,735 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T04:49:58,735 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:49:58,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:58,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:58,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T04:49:58,743 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T04:49:58,743 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=377653908, stopped=false 2024-11-14T04:49:58,743 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=44289c305c03,41663,1731559791235 2024-11-14T04:49:58,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:58,746 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T04:49:58,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:58,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:58,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:58,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:58,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:58,748 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T04:49:58,749 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:58,749 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:49:58,749 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:58,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:58,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:58,750 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '44289c305c03,45709,1731559792304' ***** 2024-11-14T04:49:58,750 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T04:49:58,750 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '44289c305c03,42037,1731559792553' ***** 2024-11-14T04:49:58,750 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T04:49:58,751 INFO [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T04:49:58,751 INFO [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T04:49:58,751 INFO [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.persistence 2024-11-14T04:49:58,751 INFO [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/bucket.persistence 2024-11-14T04:49:58,751 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T04:49:58,751 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T04:49:58,751 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T04:49:58,751 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T04:49:58,752 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T04:49:58,752 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T04:49:58,752 INFO [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T04:49:58,752 INFO [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T04:49:58,752 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T04:49:58,752 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T04:49:58,759 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:49:58,762 DEBUG [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1488): Thread RS:0;44289c305c03:45709 started persisting bucket cache to file 2024-11-14T04:49:58,762 DEBUG [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1488): Thread RS:1;44289c305c03:42037 started persisting bucket cache to file 2024-11-14T04:49:58,762 DEBUG [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:49:58,762 DEBUG [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:49:58,762 DEBUG [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:49:58,762 DEBUG [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:49:58,764 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:49:58,773 DEBUG [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T04:49:58,773 DEBUG [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T04:49:58,773 DEBUG [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731559798762 2024-11-14T04:49:58,773 DEBUG [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731559798762 2024-11-14T04:49:58,773 DEBUG [RS:1;44289c305c03:42037 {}] bucket.BucketCache(1510): Thread RS:1;44289c305c03:42037 finished persisting bucket cache to file, renaming 2024-11-14T04:49:58,773 DEBUG [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1510): Thread RS:0;44289c305c03:45709 finished persisting bucket cache to file, renaming 2024-11-14T04:49:58,774 WARN [RS:0;44289c305c03:45709 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-14T04:49:58,774 INFO [RS:0;44289c305c03:45709 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:49:58,774 INFO [RS:1;44289c305c03:42037 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:49:58,774 INFO [RS:1;44289c305c03:42037 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T04:49:58,774 INFO [RS:0;44289c305c03:45709 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T04:49:58,774 INFO [RS:1;44289c305c03:42037 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T04:49:58,774 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(3091): Received CLOSE for 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,775 INFO [RS:0;44289c305c03:45709 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T04:49:58,775 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(959): stopping server 44289c305c03,45709,1731559792304 2024-11-14T04:49:58,775 INFO [RS:0;44289c305c03:45709 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:49:58,775 INFO [RS:0;44289c305c03:45709 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;44289c305c03:45709. 2024-11-14T04:49:58,775 DEBUG [RS:0;44289c305c03:45709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:49:58,775 DEBUG [RS:0;44289c305c03:45709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:58,776 INFO [RS:0;44289c305c03:45709 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:49:58,776 INFO [RS:0;44289c305c03:45709 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:49:58,776 INFO [RS:0;44289c305c03:45709 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:49:58,776 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T04:49:58,776 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(959): stopping server 44289c305c03,42037,1731559792553 2024-11-14T04:49:58,776 INFO [RS:1;44289c305c03:42037 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:49:58,776 INFO [RS:1;44289c305c03:42037 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;44289c305c03:42037. 2024-11-14T04:49:58,776 DEBUG [RS:1;44289c305c03:42037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:49:58,776 DEBUG [RS:1;44289c305c03:42037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:58,777 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T04:49:58,777 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T04:49:58,777 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1325): Online Regions={2aca626354c0c6ec981bddd10eb8d87c=testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c.} 2024-11-14T04:49:58,777 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2aca626354c0c6ec981bddd10eb8d87c, disabling compactions & flushes 2024-11-14T04:49:58,777 DEBUG [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1351): Waiting on 2aca626354c0c6ec981bddd10eb8d87c 2024-11-14T04:49:58,777 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,777 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,777 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. after waiting 0 ms 2024-11-14T04:49:58,778 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,778 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T04:49:58,778 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T04:49:58,778 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T04:49:58,778 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T04:49:58,778 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T04:49:58,779 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T04:49:58,779 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T04:49:58,779 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T04:49:58,779 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T04:49:58,779 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-14T04:49:58,780 INFO [regionserver/44289c305c03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:49:58,781 INFO [regionserver/44289c305c03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:49:58,811 DEBUG [StoreFileCloser-2aca626354c0c6ec981bddd10eb8d87c-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/family/b950417021e04676a46ed484b035c891 2024-11-14T04:49:58,824 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aeb5ca686e314d538e69f61f7b2209a9 2024-11-14T04:49:58,824 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnRegionMove1731559796317/2aca626354c0c6ec981bddd10eb8d87c/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-14T04:49:58,825 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,825 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2aca626354c0c6ec981bddd10eb8d87c: Waiting for close lock at 1731559798776Running coprocessor pre-close hooks at 1731559798777 (+1 ms)Disabling compacts and flushes for region at 1731559798777Disabling writes for close at 1731559798777Writing region close event to WAL at 1731559798814 (+37 ms)Running coprocessor post-close hooks at 1731559798825 (+11 ms)Closed at 1731559798825 2024-11-14T04:49:58,827 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c. 2024-11-14T04:49:58,852 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/info/aeb5ca686e314d538e69f61f7b2209a9 is 205, key is testBlockEvictionOnRegionMove1731559796317,,1731559796334.2aca626354c0c6ec981bddd10eb8d87c./info:regioninfo/1731559798715/Put/seqid=0 2024-11-14T04:49:58,853 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/info/aeb5ca686e314d538e69f61f7b2209a9 2024-11-14T04:49:58,870 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 402e0af30afa4b6e9a2012c7bd75fcaf 2024-11-14T04:49:58,892 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/ns/402e0af30afa4b6e9a2012c7bd75fcaf is 43, key is default/ns:d/1731559796070/Put/seqid=0 2024-11-14T04:49:58,893 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/ns/402e0af30afa4b6e9a2012c7bd75fcaf 2024-11-14T04:49:58,926 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8fa368408c0f4590911d4a6e9de64610 2024-11-14T04:49:58,947 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/table/8fa368408c0f4590911d4a6e9de64610 is 78, key is testBlockEvictionOnRegionMove1731559796317/table:state/1731559796783/Put/seqid=0 2024-11-14T04:49:58,948 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/table/8fa368408c0f4590911d4a6e9de64610 2024-11-14T04:49:58,952 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/info/aeb5ca686e314d538e69f61f7b2209a9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/aeb5ca686e314d538e69f61f7b2209a9 2024-11-14T04:49:58,953 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/aeb5ca686e314d538e69f61f7b2209a9, delay=905 ms 2024-11-14T04:49:58,958 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/aeb5ca686e314d538e69f61f7b2209a9, entries=15, sequenceid=15, filesize=7.6 K 2024-11-14T04:49:58,959 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/ns/402e0af30afa4b6e9a2012c7bd75fcaf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/402e0af30afa4b6e9a2012c7bd75fcaf 2024-11-14T04:49:58,960 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/402e0af30afa4b6e9a2012c7bd75fcaf, delay=986 ms 2024-11-14T04:49:58,962 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/402e0af30afa4b6e9a2012c7bd75fcaf, entries=2, sequenceid=15, filesize=5.0 K 2024-11-14T04:49:58,963 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/table/8fa368408c0f4590911d4a6e9de64610 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/8fa368408c0f4590911d4a6e9de64610 2024-11-14T04:49:58,964 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/8fa368408c0f4590911d4a6e9de64610, delay=995 ms 2024-11-14T04:49:58,966 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/8fa368408c0f4590911d4a6e9de64610, entries=2, sequenceid=15, filesize=5.3 K 2024-11-14T04:49:58,967 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 188ms, sequenceid=15, compaction requested=false 2024-11-14T04:49:58,967 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T04:49:58,975 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/aeb5ca686e314d538e69f61f7b2209a9 2024-11-14T04:49:58,978 DEBUG [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T04:49:58,979 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/402e0af30afa4b6e9a2012c7bd75fcaf 2024-11-14T04:49:58,981 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/8fa368408c0f4590911d4a6e9de64610 2024-11-14T04:49:58,981 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(976): stopping server 44289c305c03,42037,1731559792553; all regions closed. 2024-11-14T04:49:58,989 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-14T04:49:58,990 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T04:49:58,990 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T04:49:58,990 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731559798778Running coprocessor pre-close hooks at 1731559798778Disabling compacts and flushes for region at 1731559798778Disabling writes for close at 1731559798779 (+1 ms)Obtaining lock to block concurrent updates at 1731559798779Preparing flush snapshotting stores in 1588230740 at 1731559798779Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731559798780 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731559798781 (+1 ms)Flushing 1588230740/info: creating writer at 1731559798781Flushing 1588230740/info: appending metadata at 1731559798850 (+69 ms)Flushing 1588230740/info: closing flushed file at 1731559798850Flushing 1588230740/ns: creating writer at 1731559798858 (+8 ms)Flushing 1588230740/ns: appending metadata at 1731559798890 (+32 ms)Flushing 1588230740/ns: closing flushed file at 1731559798890Flushing 1588230740/table: creating writer at 1731559798897 (+7 ms)Flushing 1588230740/table: appending metadata at 1731559798945 (+48 ms)Flushing 1588230740/table: closing flushed file at 1731559798945Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@551dfeb6: reopening flushed file at 1731559798951 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74ed7dbd: reopening flushed file at 1731559798959 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d234813: reopening flushed file at 1731559798963 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 188ms, sequenceid=15, compaction requested=false at 1731559798967 (+4 ms)Writing region close event to WAL at 1731559798982 (+15 ms)Running coprocessor post-close hooks at 1731559798989 (+7 ms)Closed at 1731559798990 (+1 ms) 2024-11-14T04:49:58,991 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T04:49:58,991 DEBUG [RS:1;44289c305c03:42037 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs 2024-11-14T04:49:58,992 INFO [RS:1;44289c305c03:42037 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 44289c305c03%2C42037%2C1731559792553:(num 1731559795388) 2024-11-14T04:49:58,992 DEBUG [RS:1;44289c305c03:42037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:58,992 INFO [RS:1;44289c305c03:42037 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:49:58,992 INFO [RS:1;44289c305c03:42037 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:49:58,992 INFO [RS:1;44289c305c03:42037 {}] hbase.ChoreService(370): Chore service for: regionserver/44289c305c03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T04:49:58,993 INFO [RS:1;44289c305c03:42037 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:49:58,993 INFO [RS:1;44289c305c03:42037 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:49:58,993 INFO [regionserver/44289c305c03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:49:58,993 INFO [RS:1;44289c305c03:42037 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:49:58,993 INFO [RS:1;44289c305c03:42037 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:49:58,993 INFO [RS:1;44289c305c03:42037 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42037 2024-11-14T04:49:59,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:49:59,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/44289c305c03,42037,1731559792553 2024-11-14T04:49:59,003 INFO [RS:1;44289c305c03:42037 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:49:59,006 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [44289c305c03,42037,1731559792553] 2024-11-14T04:49:59,008 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/44289c305c03,42037,1731559792553 already deleted, retry=false 2024-11-14T04:49:59,008 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 44289c305c03,42037,1731559792553 expired; onlineServers=1 2024-11-14T04:49:59,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:49:59,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42037-0x100ef948d320002, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:49:59,107 INFO [RS:1;44289c305c03:42037 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:49:59,107 INFO [RS:1;44289c305c03:42037 {}] regionserver.HRegionServer(1031): Exiting; stopping=44289c305c03,42037,1731559792553; zookeeper connection closed. 2024-11-14T04:49:59,108 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e3f52be {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e3f52be 2024-11-14T04:49:59,166 INFO [regionserver/44289c305c03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T04:49:59,166 INFO [regionserver/44289c305c03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T04:49:59,178 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(976): stopping server 44289c305c03,45709,1731559792304; all regions closed. 2024-11-14T04:49:59,182 DEBUG [RS:0;44289c305c03:45709 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs 2024-11-14T04:49:59,182 INFO [RS:0;44289c305c03:45709 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 44289c305c03%2C45709%2C1731559792304.meta:.meta(num 1731559795892) 2024-11-14T04:49:59,184 DEBUG [RS:0;44289c305c03:45709 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs 2024-11-14T04:49:59,184 INFO [RS:0;44289c305c03:45709 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 44289c305c03%2C45709%2C1731559792304:(num 1731559795379) 2024-11-14T04:49:59,184 DEBUG [RS:0;44289c305c03:45709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:49:59,184 INFO [RS:0;44289c305c03:45709 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:49:59,184 INFO [RS:0;44289c305c03:45709 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:49:59,184 INFO [RS:0;44289c305c03:45709 {}] hbase.ChoreService(370): Chore service for: regionserver/44289c305c03:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T04:49:59,184 INFO [RS:0;44289c305c03:45709 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:49:59,185 INFO [regionserver/44289c305c03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:49:59,185 INFO [RS:0;44289c305c03:45709 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45709 2024-11-14T04:49:59,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/44289c305c03,45709,1731559792304 2024-11-14T04:49:59,188 INFO [RS:0;44289c305c03:45709 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:49:59,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:49:59,189 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [44289c305c03,45709,1731559792304] 2024-11-14T04:49:59,191 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/44289c305c03,45709,1731559792304 already deleted, retry=false 2024-11-14T04:49:59,191 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 44289c305c03,45709,1731559792304 expired; onlineServers=0 2024-11-14T04:49:59,191 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '44289c305c03,41663,1731559791235' ***** 2024-11-14T04:49:59,192 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T04:49:59,192 INFO [M:0;44289c305c03:41663 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:49:59,192 INFO [M:0;44289c305c03:41663 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:49:59,192 DEBUG [M:0;44289c305c03:41663 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T04:49:59,192 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T04:49:59,192 DEBUG [M:0;44289c305c03:41663 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T04:49:59,192 DEBUG [master/44289c305c03:0:becomeActiveMaster-HFileCleaner.large.0-1731559794778 {}] cleaner.HFileCleaner(306): Exit Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.large.0-1731559794778,5,FailOnTimeoutGroup] 2024-11-14T04:49:59,192 DEBUG [master/44289c305c03:0:becomeActiveMaster-HFileCleaner.small.0-1731559794785 {}] cleaner.HFileCleaner(306): Exit Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.small.0-1731559794785,5,FailOnTimeoutGroup] 2024-11-14T04:49:59,193 INFO [M:0;44289c305c03:41663 {}] hbase.ChoreService(370): Chore service for: master/44289c305c03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T04:49:59,193 INFO [M:0;44289c305c03:41663 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:49:59,193 DEBUG [M:0;44289c305c03:41663 {}] master.HMaster(1795): Stopping service threads 2024-11-14T04:49:59,193 INFO [M:0;44289c305c03:41663 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T04:49:59,193 INFO [M:0;44289c305c03:41663 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T04:49:59,194 INFO [M:0;44289c305c03:41663 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T04:49:59,194 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T04:49:59,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T04:49:59,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,196 DEBUG [M:0;44289c305c03:41663 {}] zookeeper.ZKUtil(347): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T04:49:59,196 WARN [M:0;44289c305c03:41663 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T04:49:59,196 INFO [M:0;44289c305c03:41663 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.lastflushedseqids 2024-11-14T04:49:59,213 INFO [M:0;44289c305c03:41663 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T04:49:59,214 INFO [M:0;44289c305c03:41663 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T04:49:59,214 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T04:49:59,214 INFO [M:0;44289c305c03:41663 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,214 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,214 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T04:49:59,214 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,214 INFO [M:0;44289c305c03:41663 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-14T04:49:59,229 DEBUG [M:0;44289c305c03:41663 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 685ae185d9a8483895471b81c89bb3c2 2024-11-14T04:49:59,250 DEBUG [M:0;44289c305c03:41663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/685ae185d9a8483895471b81c89bb3c2 is 82, key is hbase:meta,,1/info:regioninfo/1731559795981/Put/seqid=0 2024-11-14T04:49:59,251 INFO [M:0;44289c305c03:41663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/685ae185d9a8483895471b81c89bb3c2 2024-11-14T04:49:59,264 DEBUG [M:0;44289c305c03:41663 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on df99898f06034ca5b216ff1f00370019 2024-11-14T04:49:59,279 DEBUG [M:0;44289c305c03:41663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df99898f06034ca5b216ff1f00370019 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731559796788/Put/seqid=0 2024-11-14T04:49:59,280 INFO [M:0;44289c305c03:41663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df99898f06034ca5b216ff1f00370019 2024-11-14T04:49:59,289 DEBUG [M:0;44289c305c03:41663 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 942fa0a62ab24836b8b7474860ee4081 2024-11-14T04:49:59,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:49:59,290 INFO [RS:0;44289c305c03:45709 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:49:59,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45709-0x100ef948d320001, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:49:59,290 INFO [RS:0;44289c305c03:45709 {}] regionserver.HRegionServer(1031): Exiting; stopping=44289c305c03,45709,1731559792304; zookeeper connection closed. 2024-11-14T04:49:59,291 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2536c33e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2536c33e 2024-11-14T04:49:59,291 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-14T04:49:59,302 DEBUG [M:0;44289c305c03:41663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/942fa0a62ab24836b8b7474860ee4081 is 69, key is 44289c305c03,42037,1731559792553/rs:state/1731559794958/Put/seqid=0 2024-11-14T04:49:59,302 INFO [M:0;44289c305c03:41663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/942fa0a62ab24836b8b7474860ee4081 2024-11-14T04:49:59,304 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/685ae185d9a8483895471b81c89bb3c2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/685ae185d9a8483895471b81c89bb3c2 2024-11-14T04:49:59,307 INFO [M:0;44289c305c03:41663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/685ae185d9a8483895471b81c89bb3c2, entries=8, sequenceid=84, filesize=5.5 K 2024-11-14T04:49:59,308 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df99898f06034ca5b216ff1f00370019 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/df99898f06034ca5b216ff1f00370019 2024-11-14T04:49:59,310 INFO [M:0;44289c305c03:41663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/df99898f06034ca5b216ff1f00370019, entries=9, sequenceid=84, filesize=6.5 K 2024-11-14T04:49:59,311 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/942fa0a62ab24836b8b7474860ee4081 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/942fa0a62ab24836b8b7474860ee4081 2024-11-14T04:49:59,313 INFO [M:0;44289c305c03:41663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/942fa0a62ab24836b8b7474860ee4081, entries=2, sequenceid=84, filesize=5.1 K 2024-11-14T04:49:59,314 INFO [M:0;44289c305c03:41663 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=84, compaction requested=false 2024-11-14T04:49:59,319 INFO [M:0;44289c305c03:41663 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,320 DEBUG [M:0;44289c305c03:41663 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731559799214Disabling compacts and flushes for region at 1731559799214Disabling writes for close at 1731559799214Obtaining lock to block concurrent updates at 1731559799214Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731559799214Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731559799215 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731559799215Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731559799216 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731559799249 (+33 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731559799249Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731559799254 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731559799279 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731559799279Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731559799282 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731559799301 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731559799301Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41ade2c5: reopening flushed file at 1731559799304 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f802495: reopening flushed file at 1731559799307 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8c1f8f4: reopening flushed file at 1731559799310 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=84, compaction requested=false at 1731559799314 (+4 ms)Writing region close event to WAL at 1731559799319 (+5 ms)Closed at 1731559799319 2024-11-14T04:49:59,323 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:49:59,324 INFO [M:0;44289c305c03:41663 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T04:49:59,324 INFO [M:0;44289c305c03:41663 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41663 2024-11-14T04:49:59,324 INFO [M:0;44289c305c03:41663 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:49:59,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:49:59,434 INFO [M:0;44289c305c03:41663 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:49:59,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41663-0x100ef948d320000, quorum=127.0.0.1:61305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:49:59,439 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T04:49:59,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T04:49:59,466 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,475 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/44289c305c03:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/44289c305c03:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731559798960 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731559798696 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/44289c305c03:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1731559798953 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731559797152 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/44289c305c03:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=767 (was 761) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8362 (was 7684) - AvailableMemoryMB LEAK? - 2024-11-14T04:49:59,481 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=767, ProcessCount=11, AvailableMemoryMB=8362 2024-11-14T04:49:59,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/hadoop-log-dir so I do NOT create it in target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0 2024-11-14T04:49:59,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/hadoop-tmp-dir so I do NOT create it in target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0 2024-11-14T04:49:59,482 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,485 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/cluster_d34ac232-2203-8f54-12fe-44247682f7c3, deleteOnExit=true 2024-11-14T04:49:59,492 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/cluster_d34ac232-2203-8f54-12fe-44247682f7c3/zookeeper_0, clientPort=50033, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/cluster_d34ac232-2203-8f54-12fe-44247682f7c3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/cluster_d34ac232-2203-8f54-12fe-44247682f7c3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T04:49:59,493 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50033 2024-11-14T04:49:59,508 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 with version=8 2024-11-14T04:49:59,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/hbase-staging 2024-11-14T04:49:59,511 INFO [Time-limited test {}] client.ConnectionUtils(128): master/44289c305c03:0 server-side Connection retries=45 2024-11-14T04:49:59,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,512 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:49:59,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:49:59,512 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T04:49:59,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:49:59,513 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40261 2024-11-14T04:49:59,515 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40261 connecting to ZooKeeper ensemble=127.0.0.1:50033 2024-11-14T04:49:59,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402610x0, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:49:59,526 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40261-0x100ef94af6b0000 connected 2024-11-14T04:49:59,550 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,551 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,552 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:59,552 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643, hbase.cluster.distributed=false 2024-11-14T04:49:59,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:49:59,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40261 2024-11-14T04:49:59,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40261 2024-11-14T04:49:59,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40261 2024-11-14T04:49:59,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40261 2024-11-14T04:49:59,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40261 2024-11-14T04:49:59,575 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/44289c305c03:0 server-side Connection retries=45 2024-11-14T04:49:59,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,575 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:49:59,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:49:59,575 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:49:59,576 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:49:59,577 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34263 2024-11-14T04:49:59,579 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34263 connecting to ZooKeeper ensemble=127.0.0.1:50033 2024-11-14T04:49:59,579 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,579 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342630x0, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:49:59,585 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:342630x0, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:59,585 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34263-0x100ef94af6b0001 connected 2024-11-14T04:49:59,585 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:49:59,589 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.cache 2024-11-14T04:49:59,589 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T04:49:59,589 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T04:49:59,589 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T04:49:59,597 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T04:49:59,597 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T04:49:59,597 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T04:49:59,598 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T04:49:59,598 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence is complete. 2024-11-14T04:49:59,599 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:49:59,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T04:49:59,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:49:59,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34263 2024-11-14T04:49:59,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34263 2024-11-14T04:49:59,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34263 2024-11-14T04:49:59,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34263 2024-11-14T04:49:59,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34263 2024-11-14T04:49:59,623 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/44289c305c03:0 server-side Connection retries=45 2024-11-14T04:49:59,623 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,623 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,623 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:49:59,623 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:49:59,623 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:49:59,623 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:49:59,624 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:49:59,624 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38537 2024-11-14T04:49:59,626 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38537 connecting to ZooKeeper ensemble=127.0.0.1:50033 2024-11-14T04:49:59,626 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,626 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385370x0, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:49:59,630 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38537-0x100ef94af6b0002 connected 2024-11-14T04:49:59,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:49:59,630 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:49:59,631 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.cache 2024-11-14T04:49:59,631 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T04:49:59,631 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T04:49:59,631 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T04:49:59,632 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T04:49:59,632 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T04:49:59,632 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T04:49:59,632 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:49:59,632 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T04:49:59,632 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence is complete. 2024-11-14T04:49:59,633 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T04:49:59,634 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:49:59,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38537 2024-11-14T04:49:59,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38537 2024-11-14T04:49:59,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38537 2024-11-14T04:49:59,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38537 2024-11-14T04:49:59,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38537 2024-11-14T04:49:59,660 DEBUG [M:0;44289c305c03:40261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;44289c305c03:40261 2024-11-14T04:49:59,660 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/44289c305c03,40261,1731559799511 2024-11-14T04:49:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:59,663 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/44289c305c03,40261,1731559799511 2024-11-14T04:49:59,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T04:49:59,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T04:49:59,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,667 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T04:49:59,668 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/44289c305c03,40261,1731559799511 from backup master directory 2024-11-14T04:49:59,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:59,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/44289c305c03,40261,1731559799511 2024-11-14T04:49:59,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:59,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:49:59,671 WARN [master/44289c305c03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:49:59,671 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=44289c305c03,40261,1731559799511 2024-11-14T04:49:59,673 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/hbase.id] with ID: 389c1722-e708-43f8-a99b-a2edafafcdd5 2024-11-14T04:49:59,673 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.tmp/hbase.id 2024-11-14T04:49:59,681 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/hbase.id] 2024-11-14T04:49:59,725 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:49:59,726 INFO [master/44289c305c03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T04:49:59,727 INFO [master/44289c305c03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-14T04:49:59,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,737 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T04:49:59,738 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T04:49:59,739 INFO [master/44289c305c03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:49:59,764 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store 2024-11-14T04:49:59,772 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:59,772 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T04:49:59,772 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,772 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,772 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T04:49:59,772 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,772 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:49:59,773 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731559799772Disabling compacts and flushes for region at 1731559799772Disabling writes for close at 1731559799772Writing region close event to WAL at 1731559799772Closed at 1731559799772 2024-11-14T04:49:59,773 WARN [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/.initializing 2024-11-14T04:49:59,773 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/WALs/44289c305c03,40261,1731559799511 2024-11-14T04:49:59,780 INFO [master/44289c305c03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:49:59,786 INFO [master/44289c305c03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=44289c305c03%2C40261%2C1731559799511, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/WALs/44289c305c03,40261,1731559799511, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/oldWALs, maxLogs=10 2024-11-14T04:49:59,803 INFO [master/44289c305c03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/WALs/44289c305c03,40261,1731559799511/44289c305c03%2C40261%2C1731559799511.1731559799786 2024-11-14T04:49:59,804 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:49:59,804 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:49:59,804 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:59,804 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,804 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T04:49:59,811 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:59,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T04:49:59,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:59,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T04:49:59,825 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:59,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T04:49:59,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:49:59,830 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,830 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,830 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,831 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,831 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,831 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:49:59,831 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:49:59,839 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:49:59,839 INFO [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69401244, jitterRate=0.03415912389755249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:49:59,840 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731559799805Initializing all the Stores at 1731559799805Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559799805Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559799805Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559799805Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559799805Cleaning up temporary data from old regions at 1731559799831 (+26 ms)Region opened successfully at 1731559799840 (+9 ms) 2024-11-14T04:49:59,841 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T04:49:59,852 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ed31a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=44289c305c03/172.17.0.2:0 2024-11-14T04:49:59,854 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T04:49:59,854 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T04:49:59,854 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T04:49:59,855 INFO [master/44289c305c03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T04:49:59,855 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T04:49:59,855 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T04:49:59,855 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T04:49:59,859 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T04:49:59,860 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T04:49:59,862 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T04:49:59,862 INFO [master/44289c305c03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T04:49:59,863 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T04:49:59,865 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T04:49:59,865 INFO [master/44289c305c03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T04:49:59,866 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T04:49:59,868 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T04:49:59,869 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T04:49:59,870 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T04:49:59,873 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T04:49:59,874 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T04:49:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:49:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,877 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=44289c305c03,40261,1731559799511, sessionid=0x100ef94af6b0000, setting cluster-up flag (Was=false) 2024-11-14T04:49:59,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,887 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T04:49:59,888 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=44289c305c03,40261,1731559799511 2024-11-14T04:49:59,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:49:59,903 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T04:49:59,904 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=44289c305c03,40261,1731559799511 2024-11-14T04:49:59,905 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T04:49:59,907 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T04:49:59,907 INFO [master/44289c305c03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T04:49:59,907 INFO [master/44289c305c03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T04:49:59,908 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 44289c305c03,40261,1731559799511 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/44289c305c03:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/44289c305c03:0, corePoolSize=10, maxPoolSize=10 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/44289c305c03:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:49:59,910 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:49:59,919 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:49:59,919 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T04:49:59,919 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,920 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T04:49:59,921 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731559829921 2024-11-14T04:49:59,922 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T04:49:59,922 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T04:49:59,922 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T04:49:59,922 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T04:49:59,922 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T04:49:59,922 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T04:49:59,922 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:59,932 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T04:49:59,932 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T04:49:59,933 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T04:49:59,935 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T04:49:59,935 INFO [master/44289c305c03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T04:49:59,936 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.large.0-1731559799935,5,FailOnTimeoutGroup] 2024-11-14T04:49:59,936 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.small.0-1731559799936,5,FailOnTimeoutGroup] 2024-11-14T04:49:59,936 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:59,936 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T04:49:59,936 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:59,936 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T04:49:59,944 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(746): ClusterId : 389c1722-e708-43f8-a99b-a2edafafcdd5 2024-11-14T04:49:59,944 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(746): ClusterId : 389c1722-e708-43f8-a99b-a2edafafcdd5 2024-11-14T04:49:59,944 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:49:59,944 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:49:59,945 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T04:49:59,946 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:49:59,947 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:49:59,947 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:49:59,947 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:49:59,947 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:49:59,950 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:49:59,950 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:49:59,951 DEBUG [RS:0;44289c305c03:34263 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d0b137, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=44289c305c03/172.17.0.2:0 2024-11-14T04:49:59,951 DEBUG [RS:1;44289c305c03:38537 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e0835b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=44289c305c03/172.17.0.2:0 2024-11-14T04:49:59,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:49:59,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T04:49:59,965 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;44289c305c03:34263 2024-11-14T04:49:59,965 INFO [RS:0;44289c305c03:34263 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:49:59,965 INFO [RS:0;44289c305c03:34263 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:49:59,966 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:49:59,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T04:49:59,967 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(2659): reportForDuty to master=44289c305c03,40261,1731559799511 with port=34263, startcode=1731559799574 2024-11-14T04:49:59,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,967 DEBUG [RS:0;44289c305c03:34263 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:49:59,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:59,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T04:49:59,969 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;44289c305c03:38537 2024-11-14T04:49:59,969 INFO [RS:1;44289c305c03:38537 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:49:59,969 INFO [RS:1;44289c305c03:38537 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:49:59,969 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:49:59,971 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(2659): reportForDuty to master=44289c305c03,40261,1731559799511 with port=38537, startcode=1731559799622 2024-11-14T04:49:59,971 DEBUG [RS:1;44289c305c03:38537 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:49:59,972 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36837, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:49:59,973 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40261 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 44289c305c03,34263,1731559799574 2024-11-14T04:49:59,973 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40261 {}] master.ServerManager(517): Registering regionserver=44289c305c03,34263,1731559799574 2024-11-14T04:49:59,974 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T04:49:59,974 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:59,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T04:49:59,975 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:49:59,975 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:49:59,975 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T04:49:59,975 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:49:59,976 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40261 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 44289c305c03,38537,1731559799622 2024-11-14T04:49:59,976 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40261 {}] master.ServerManager(517): Registering regionserver=44289c305c03,38537,1731559799622 2024-11-14T04:49:59,978 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:49:59,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:49:59,978 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T04:49:59,978 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:49:59,979 DEBUG [RS:0;44289c305c03:34263 {}] zookeeper.ZKUtil(111): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/44289c305c03,34263,1731559799574 2024-11-14T04:49:59,979 WARN [RS:0;44289c305c03:34263 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:49:59,979 INFO [RS:0;44289c305c03:34263 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:49:59,979 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,34263,1731559799574 2024-11-14T04:49:59,986 DEBUG [RS:1;44289c305c03:38537 {}] zookeeper.ZKUtil(111): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/44289c305c03,38537,1731559799622 2024-11-14T04:49:59,986 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T04:49:59,986 WARN [RS:1;44289c305c03:38537 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:49:59,986 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,986 INFO [RS:1;44289c305c03:38537 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:49:59,986 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,38537,1731559799622 2024-11-14T04:49:59,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:59,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T04:49:59,988 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [44289c305c03,38537,1731559799622] 2024-11-14T04:49:59,988 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [44289c305c03,34263,1731559799574] 2024-11-14T04:49:59,990 INFO [RS:0;44289c305c03:34263 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:49:59,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T04:49:59,992 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:49:59,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:49:59,993 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T04:49:59,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:49:59,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:49:59,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T04:49:59,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T04:49:59,994 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:49:59,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T04:49:59,997 INFO [RS:0;44289c305c03:34263 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:49:59,999 INFO [RS:0;44289c305c03:34263 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:49:59,999 INFO [RS:1;44289c305c03:38537 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:49:59,999 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,001 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:50:00,002 INFO [RS:0;44289c305c03:34263 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:50:00,002 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,002 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/44289c305c03:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,003 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,004 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,004 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,004 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,004 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:50:00,004 DEBUG [RS:0;44289c305c03:34263 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:50:00,005 INFO [RS:1;44289c305c03:38537 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:50:00,007 INFO [RS:1;44289c305c03:38537 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:50:00,007 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,007 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,007 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,007 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,007 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:50:00,007 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64663032, jitterRate=-0.0364457368850708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:50:00,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731559799961Initializing all the Stores at 1731559799961Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559799961Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559799961Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559799961Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559799961Cleaning up temporary data from old regions at 1731559799994 (+33 ms)Region opened successfully at 1731559800008 (+14 ms) 2024-11-14T04:50:00,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T04:50:00,008 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T04:50:00,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T04:50:00,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T04:50:00,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T04:50:00,009 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,009 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,009 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,34263,1731559799574-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:50:00,009 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:50:00,009 INFO [RS:1;44289c305c03:38537 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:50:00,010 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,010 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,010 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,010 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,010 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/44289c305c03:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:00,011 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:50:00,012 DEBUG [RS:1;44289c305c03:38537 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:50:00,012 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T04:50:00,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731559800008Disabling compacts and flushes for region at 1731559800008Disabling writes for close at 1731559800008Writing region close event to WAL at 1731559800012 (+4 ms)Closed at 1731559800012 2024-11-14T04:50:00,012 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,012 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,013 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,013 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,013 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,013 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,38537,1731559799622-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:50:00,013 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:50:00,013 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T04:50:00,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T04:50:00,016 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T04:50:00,017 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T04:50:00,030 INFO [RS:1;44289c305c03:38537 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:50:00,030 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,38537,1731559799622-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,030 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,030 INFO [RS:1;44289c305c03:38537 {}] regionserver.Replication(171): 44289c305c03,38537,1731559799622 started 2024-11-14T04:50:00,035 INFO [RS:0;44289c305c03:34263 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:50:00,035 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,34263,1731559799574-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,036 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,036 INFO [RS:0;44289c305c03:34263 {}] regionserver.Replication(171): 44289c305c03,34263,1731559799574 started 2024-11-14T04:50:00,047 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,047 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1482): Serving as 44289c305c03,38537,1731559799622, RpcServer on 44289c305c03/172.17.0.2:38537, sessionid=0x100ef94af6b0002 2024-11-14T04:50:00,048 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:50:00,048 DEBUG [RS:1;44289c305c03:38537 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 44289c305c03,38537,1731559799622 2024-11-14T04:50:00,048 DEBUG [RS:1;44289c305c03:38537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,38537,1731559799622' 2024-11-14T04:50:00,048 DEBUG [RS:1;44289c305c03:38537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:50:00,049 DEBUG [RS:1;44289c305c03:38537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:50:00,050 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:50:00,050 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:50:00,050 DEBUG [RS:1;44289c305c03:38537 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 44289c305c03,38537,1731559799622 2024-11-14T04:50:00,050 DEBUG [RS:1;44289c305c03:38537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,38537,1731559799622' 2024-11-14T04:50:00,050 DEBUG [RS:1;44289c305c03:38537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:50:00,050 DEBUG [RS:1;44289c305c03:38537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:50:00,051 DEBUG [RS:1;44289c305c03:38537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:50:00,051 INFO [RS:1;44289c305c03:38537 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:50:00,051 INFO [RS:1;44289c305c03:38537 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:50:00,059 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,059 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1482): Serving as 44289c305c03,34263,1731559799574, RpcServer on 44289c305c03/172.17.0.2:34263, sessionid=0x100ef94af6b0001 2024-11-14T04:50:00,059 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:50:00,059 DEBUG [RS:0;44289c305c03:34263 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 44289c305c03,34263,1731559799574 2024-11-14T04:50:00,059 DEBUG [RS:0;44289c305c03:34263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,34263,1731559799574' 2024-11-14T04:50:00,059 DEBUG [RS:0;44289c305c03:34263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:50:00,060 DEBUG [RS:0;44289c305c03:34263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:50:00,061 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:50:00,061 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:50:00,061 DEBUG [RS:0;44289c305c03:34263 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 44289c305c03,34263,1731559799574 2024-11-14T04:50:00,061 DEBUG [RS:0;44289c305c03:34263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,34263,1731559799574' 2024-11-14T04:50:00,061 DEBUG [RS:0;44289c305c03:34263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:50:00,062 DEBUG [RS:0;44289c305c03:34263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:50:00,063 DEBUG [RS:0;44289c305c03:34263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:50:00,063 INFO [RS:0;44289c305c03:34263 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:50:00,063 INFO [RS:0;44289c305c03:34263 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:50:00,152 INFO [RS:1;44289c305c03:38537 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:50:00,152 INFO [RS:1;44289c305c03:38537 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=44289c305c03%2C38537%2C1731559799622, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,38537,1731559799622, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs, maxLogs=55 2024-11-14T04:50:00,164 INFO [RS:0;44289c305c03:34263 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:50:00,165 INFO [RS:0;44289c305c03:34263 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=44289c305c03%2C34263%2C1731559799574, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,34263,1731559799574, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs, maxLogs=55 2024-11-14T04:50:00,168 WARN [44289c305c03:40261 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T04:50:00,175 INFO [RS:1;44289c305c03:38537 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,38537,1731559799622/44289c305c03%2C38537%2C1731559799622.1731559800154 2024-11-14T04:50:00,183 DEBUG [RS:1;44289c305c03:38537 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:50:00,192 INFO [RS:0;44289c305c03:34263 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,34263,1731559799574/44289c305c03%2C34263%2C1731559799574.1731559800166 2024-11-14T04:50:00,192 DEBUG [RS:0;44289c305c03:34263 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:50:00,418 DEBUG [44289c305c03:40261 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T04:50:00,418 DEBUG [44289c305c03:40261 {}] balancer.BalancerClusterState(204): Hosts are {44289c305c03=0} racks are {/default-rack=0} 2024-11-14T04:50:00,419 DEBUG [44289c305c03:40261 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T04:50:00,419 DEBUG [44289c305c03:40261 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T04:50:00,419 DEBUG [44289c305c03:40261 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T04:50:00,419 DEBUG [44289c305c03:40261 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T04:50:00,419 INFO [44289c305c03:40261 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T04:50:00,419 INFO [44289c305c03:40261 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T04:50:00,419 DEBUG [44289c305c03:40261 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T04:50:00,420 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=44289c305c03,34263,1731559799574 2024-11-14T04:50:00,421 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 44289c305c03,34263,1731559799574, state=OPENING 2024-11-14T04:50:00,423 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T04:50:00,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:50:00,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:50:00,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:50:00,425 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T04:50:00,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:50:00,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:50:00,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=44289c305c03,34263,1731559799574}] 2024-11-14T04:50:00,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:50:00,579 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:50:00,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41053, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:50:00,585 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T04:50:00,585 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:50:00,586 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T04:50:00,587 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=44289c305c03%2C34263%2C1731559799574.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,34263,1731559799574, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs, maxLogs=55 2024-11-14T04:50:00,607 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,34263,1731559799574/44289c305c03%2C34263%2C1731559799574.meta.1731559800588.meta 2024-11-14T04:50:00,608 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:50:00,609 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:50:00,609 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T04:50:00,609 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T04:50:00,609 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T04:50:00,609 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T04:50:00,610 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:50:00,610 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T04:50:00,610 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T04:50:00,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T04:50:00,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T04:50:00,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:50:00,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:50:00,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T04:50:00,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T04:50:00,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:50:00,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:50:00,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T04:50:00,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T04:50:00,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:50:00,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:50:00,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T04:50:00,615 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T04:50:00,615 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:50:00,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:50:00,615 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T04:50:00,616 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:50:00,617 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740 2024-11-14T04:50:00,617 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T04:50:00,617 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T04:50:00,618 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:50:00,619 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T04:50:00,619 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67601050, jitterRate=0.0073341429233551025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:50:00,619 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T04:50:00,620 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731559800610Writing region info on filesystem at 1731559800610Initializing all the Stores at 1731559800610Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559800610Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559800611 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559800611Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731559800611Cleaning up temporary data from old regions at 1731559800617 (+6 ms)Running coprocessor post-open hooks at 1731559800619 (+2 ms)Region opened successfully at 1731559800620 (+1 ms) 2024-11-14T04:50:00,621 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731559800579 2024-11-14T04:50:00,624 DEBUG [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T04:50:00,624 INFO [RS_OPEN_META-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T04:50:00,625 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=44289c305c03,34263,1731559799574 2024-11-14T04:50:00,625 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 44289c305c03,34263,1731559799574, state=OPEN 2024-11-14T04:50:00,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:50:00,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:50:00,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:50:00,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:50:00,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:50:00,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:50:00,628 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=44289c305c03,34263,1731559799574 2024-11-14T04:50:00,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T04:50:00,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=44289c305c03,34263,1731559799574 in 202 msec 2024-11-14T04:50:00,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T04:50:00,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 616 msec 2024-11-14T04:50:00,634 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:50:00,634 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T04:50:00,637 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T04:50:00,638 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=44289c305c03,34263,1731559799574, seqNum=-1] 2024-11-14T04:50:00,638 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:50:00,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54055, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:50:00,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 738 msec 2024-11-14T04:50:00,645 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731559800645, completionTime=-1 2024-11-14T04:50:00,645 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-14T04:50:00,645 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T04:50:00,647 INFO [master/44289c305c03:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T04:50:00,647 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731559860647 2024-11-14T04:50:00,647 INFO [master/44289c305c03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731559920647 2024-11-14T04:50:00,647 INFO [master/44289c305c03:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-14T04:50:00,648 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,40261,1731559799511-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,648 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,40261,1731559799511-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,648 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,40261,1731559799511-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,648 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-44289c305c03:40261, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,648 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,648 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,651 DEBUG [master/44289c305c03:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.982sec 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,40261,1731559799511-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:50:00,653 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,40261,1731559799511-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T04:50:00,654 DEBUG [master/44289c305c03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T04:50:00,654 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T04:50:00,654 INFO [master/44289c305c03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,40261,1731559799511-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:00,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e576d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:50:00,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 44289c305c03,40261,-1 for getting cluster id 2024-11-14T04:50:00,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T04:50:00,746 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '389c1722-e708-43f8-a99b-a2edafafcdd5' 2024-11-14T04:50:00,747 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T04:50:00,747 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "389c1722-e708-43f8-a99b-a2edafafcdd5" 2024-11-14T04:50:00,747 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37db8a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:50:00,748 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [44289c305c03,40261,-1] 2024-11-14T04:50:00,748 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T04:50:00,748 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:00,750 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T04:50:00,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a912745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:50:00,752 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T04:50:00,755 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=44289c305c03,34263,1731559799574, seqNum=-1] 2024-11-14T04:50:00,756 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:50:00,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:50:00,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=44289c305c03,40261,1731559799511 2024-11-14T04:50:00,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T04:50:00,763 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 44289c305c03,40261,1731559799511 2024-11-14T04:50:00,763 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78eb0515 2024-11-14T04:50:00,764 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T04:50:00,766 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T04:50:00,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731559800762', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T04:50:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 2024-11-14T04:50:00,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T04:50:00,769 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:50:00,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731559800762" procId is: 4 2024-11-14T04:50:00,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T04:50:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:50:00,799 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e76b4f363a336ed8ed2087be33abe67, NAME => 'testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731559800762', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:50:00,811 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:50:00,811 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(1722): Closing 5e76b4f363a336ed8ed2087be33abe67, disabling compactions & flushes 2024-11-14T04:50:00,811 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:00,811 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:00,811 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. after waiting 0 ms 2024-11-14T04:50:00,811 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:00,811 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:00,811 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731559800762-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e76b4f363a336ed8ed2087be33abe67: Waiting for close lock at 1731559800811Disabling compacts and flushes for region at 1731559800811Disabling writes for close at 1731559800811Writing region close event to WAL at 1731559800811Closed at 1731559800811 2024-11-14T04:50:00,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T04:50:00,813 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731559800813"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731559800813"}]},"ts":"1731559800813"} 2024-11-14T04:50:00,823 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T04:50:00,824 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T04:50:00,827 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731559800762","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731559800824"}]},"ts":"1731559800824"} 2024-11-14T04:50:00,834 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731559800762, state=ENABLING in hbase:meta 2024-11-14T04:50:00,834 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {44289c305c03=0} racks are {/default-rack=0} 2024-11-14T04:50:00,835 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T04:50:00,835 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T04:50:00,835 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T04:50:00,835 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T04:50:00,835 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T04:50:00,835 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T04:50:00,835 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T04:50:00,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN}] 2024-11-14T04:50:00,839 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN 2024-11-14T04:50:00,840 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN; state=OFFLINE, location=44289c305c03,38537,1731559799622; forceNewPlan=false, retain=false 2024-11-14T04:50:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:50:00,991 INFO [44289c305c03:40261 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T04:50:00,991 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5e76b4f363a336ed8ed2087be33abe67, regionState=OPENING, regionLocation=44289c305c03,38537,1731559799622 2024-11-14T04:50:00,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN because future has completed 2024-11-14T04:50:00,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e76b4f363a336ed8ed2087be33abe67, server=44289c305c03,38537,1731559799622}] 2024-11-14T04:50:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:50:01,150 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:50:01,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42871, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:50:01,155 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:01,156 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e76b4f363a336ed8ed2087be33abe67, NAME => 'testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:50:01,156 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731559800762 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,156 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:50:01,157 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,157 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,158 INFO [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,162 INFO [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e76b4f363a336ed8ed2087be33abe67 columnFamilyName family 2024-11-14T04:50:01,162 DEBUG [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:50:01,162 INFO [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] regionserver.HStore(327): Store=5e76b4f363a336ed8ed2087be33abe67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:50:01,163 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,163 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,163 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,164 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,164 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,165 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,173 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:50:01,173 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5e76b4f363a336ed8ed2087be33abe67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65454745, jitterRate=-0.02464829385280609}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:50:01,173 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:01,175 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5e76b4f363a336ed8ed2087be33abe67: Running coprocessor pre-open hook at 1731559801157Writing region info on filesystem at 1731559801157Initializing all the Stores at 1731559801157Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559801157Cleaning up temporary data from old regions at 1731559801164 (+7 ms)Running coprocessor post-open hooks at 1731559801173 (+9 ms)Region opened successfully at 1731559801175 (+2 ms) 2024-11-14T04:50:01,176 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67., pid=6, masterSystemTime=1731559801150 2024-11-14T04:50:01,179 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:01,179 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:01,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5e76b4f363a336ed8ed2087be33abe67, regionState=OPEN, openSeqNum=2, regionLocation=44289c305c03,38537,1731559799622 2024-11-14T04:50:01,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e76b4f363a336ed8ed2087be33abe67, server=44289c305c03,38537,1731559799622 because future has completed 2024-11-14T04:50:01,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T04:50:01,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e76b4f363a336ed8ed2087be33abe67, server=44289c305c03,38537,1731559799622 in 189 msec 2024-11-14T04:50:01,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T04:50:01,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN in 353 msec 2024-11-14T04:50:01,191 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T04:50:01,191 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731559800762","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731559801191"}]},"ts":"1731559801191"} 2024-11-14T04:50:01,194 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731559800762, state=ENABLED in hbase:meta 2024-11-14T04:50:01,195 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T04:50:01,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731559800762 in 428 msec 2024-11-14T04:50:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:50:01,401 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731559800762 completed 2024-11-14T04:50:01,401 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731559800762 get assigned. Timeout = 60000ms 2024-11-14T04:50:01,401 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T04:50:01,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731559800762 assigned to meta. Checking AM states. 2024-11-14T04:50:01,406 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T04:50:01,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731559800762 assigned. 2024-11-14T04:50:01,410 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731559800762', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67., hostname=44289c305c03,38537,1731559799622, seqNum=2] 2024-11-14T04:50:01,411 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:50:01,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:50:01,418 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5e76b4f363a336ed8ed2087be33abe67 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-14T04:50:01,429 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c93b926d4ebf47df9d74c26de41de93a 2024-11-14T04:50:01,444 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T04:50:01,454 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/.tmp/family/c93b926d4ebf47df9d74c26de41de93a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-14T04:50:01,455 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/.tmp/family/c93b926d4ebf47df9d74c26de41de93a 2024-11-14T04:50:01,459 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/.tmp/family/c93b926d4ebf47df9d74c26de41de93a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a 2024-11-14T04:50:01,461 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a, delay=925 ms 2024-11-14T04:50:01,463 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-14T04:50:01,464 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 5e76b4f363a336ed8ed2087be33abe67 in 46ms, sequenceid=6, compaction requested=false 2024-11-14T04:50:01,464 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731559800762' 2024-11-14T04:50:01,466 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5e76b4f363a336ed8ed2087be33abe67: 2024-11-14T04:50:01,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T04:50:01,885 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T04:50:01,887 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T04:50:01,887 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T04:50:01,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T04:50:01,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T04:50:01,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731559800762 2024-11-14T04:50:01,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731559800762 Metrics about Tables on a single HBase RegionServer 2024-11-14T04:50:02,387 DEBUG [hfile-prefetch-1731559798696 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c93b926d4ebf47df9d74c26de41de93a_0. 2024-11-14T04:50:02,388 DEBUG [hfile-prefetch-1731559798696 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c93b926d4ebf47df9d74c26de41de93a_117. 2024-11-14T04:50:02,389 DEBUG [hfile-prefetch-1731559798696 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a, with total blocks 2, and data blocks 1 2024-11-14T04:50:02,389 DEBUG [hfile-prefetch-1731559798696 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T04:50:02,389 DEBUG [hfile-prefetch-1731559798696 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T04:50:02,389 DEBUG [hfile-prefetch-1731559798696 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a. Will sleep 100ms and try the verification again. 2024-11-14T04:50:02,395 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T04:50:02,395 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T04:50:02,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:50:02,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:50:02,400 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T04:50:02,400 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731559802395 2024-11-14T04:50:02,400 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T04:50:02,467 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;44289c305c03:38537,5,FailOnTimeoutGroup] 2024-11-14T04:50:02,467 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '44289c305c03,38537,1731559799622' ***** 2024-11-14T04:50:02,467 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-14T04:50:02,467 INFO [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T04:50:02,468 INFO [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence 2024-11-14T04:50:02,468 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T04:50:02,468 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T04:50:02,468 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T04:50:02,468 INFO [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T04:50:02,468 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T04:50:02,478 DEBUG [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1488): Thread RS:1;44289c305c03:38537 started persisting bucket cache to file 2024-11-14T04:50:02,478 DEBUG [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:50:02,479 DEBUG [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:50:02,486 DEBUG [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-14T04:50:02,486 DEBUG [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731559802478 2024-11-14T04:50:02,487 DEBUG [RS:1;44289c305c03:38537 {}] bucket.BucketCache(1510): Thread RS:1;44289c305c03:38537 finished persisting bucket cache to file, renaming 2024-11-14T04:50:02,487 INFO [RS:1;44289c305c03:38537 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:50:02,487 INFO [RS:1;44289c305c03:38537 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T04:50:02,487 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:50:02,487 INFO [RS:1;44289c305c03:38537 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T04:50:02,487 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:02,487 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(959): stopping server 44289c305c03,38537,1731559799622 2024-11-14T04:50:02,487 INFO [RS:1;44289c305c03:38537 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:50:02,488 INFO [RS:1;44289c305c03:38537 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;44289c305c03:38537. 2024-11-14T04:50:02,488 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T04:50:02,488 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e76b4f363a336ed8ed2087be33abe67, disabling compactions & flushes 2024-11-14T04:50:02,488 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:02,488 DEBUG [RS:1;44289c305c03:38537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:50:02,488 DEBUG [RS:1;44289c305c03:38537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:02,488 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:02,488 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. after waiting 0 ms 2024-11-14T04:50:02,488 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:02,488 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T04:50:02,488 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1325): Online Regions={5e76b4f363a336ed8ed2087be33abe67=testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.} 2024-11-14T04:50:02,488 DEBUG [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1351): Waiting on 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:02,489 DEBUG [StoreFileCloser-5e76b4f363a336ed8ed2087be33abe67-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a 2024-11-14T04:50:02,491 WARN [hfile-prefetch-1731559798696 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-14T04:50:02,492 DEBUG [hfile-prefetch-1731559798696 {}] hfile.PrefetchExecutor(121): Prefetch completed for c93b926d4ebf47df9d74c26de41de93a 2024-11-14T04:50:02,495 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T04:50:02,495 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:02,495 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e76b4f363a336ed8ed2087be33abe67: Waiting for close lock at 1731559802488Running coprocessor pre-close hooks at 1731559802488Disabling compacts and flushes for region at 1731559802488Disabling writes for close at 1731559802488Writing region close event to WAL at 1731559802489 (+1 ms)Running coprocessor post-close hooks at 1731559802495 (+6 ms)Closed at 1731559802495 2024-11-14T04:50:02,496 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:02,520 INFO [regionserver/44289c305c03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:50:02,688 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(976): stopping server 44289c305c03,38537,1731559799622; all regions closed. 2024-11-14T04:50:02,690 DEBUG [RS:1;44289c305c03:38537 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 44289c305c03%2C38537%2C1731559799622:(num 1731559800154) 2024-11-14T04:50:02,691 DEBUG [RS:1;44289c305c03:38537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] hbase.ChoreService(370): Chore service for: regionserver/44289c305c03:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:50:02,691 INFO [regionserver/44289c305c03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:50:02,691 INFO [RS:1;44289c305c03:38537 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38537 2024-11-14T04:50:02,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/44289c305c03,38537,1731559799622 2024-11-14T04:50:02,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:50:02,696 INFO [RS:1;44289c305c03:38537 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:50:02,699 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [44289c305c03,38537,1731559799622] 2024-11-14T04:50:02,701 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/44289c305c03,38537,1731559799622 already deleted, retry=false 2024-11-14T04:50:02,701 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 44289c305c03,38537,1731559799622 on 44289c305c03,40261,1731559799511 2024-11-14T04:50:02,707 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 44289c305c03,38537,1731559799622, splitWal=true, meta=false 2024-11-14T04:50:02,710 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 44289c305c03,38537,1731559799622, splitWal=true, meta=false 2024-11-14T04:50:02,710 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 44289c305c03,38537,1731559799622 (carryingMeta=false) 44289c305c03,38537,1731559799622/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3738dcdf[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T04:50:02,712 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 44289c305c03,38537,1731559799622 had 1 regions 2024-11-14T04:50:02,713 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 44289c305c03,38537,1731559799622, splitWal=true, meta=false, isMeta: false 2024-11-14T04:50:02,713 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,38537,1731559799622-splitting 2024-11-14T04:50:02,714 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,38537,1731559799622-splitting dir is empty, no logs to split. 2024-11-14T04:50:02,714 INFO [PEWorker-2 {}] master.SplitWALManager(105): 44289c305c03,38537,1731559799622 WAL count=0, meta=false 2024-11-14T04:50:02,717 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,38537,1731559799622-splitting dir is empty, no logs to split. 2024-11-14T04:50:02,717 INFO [PEWorker-2 {}] master.SplitWALManager(105): 44289c305c03,38537,1731559799622 WAL count=0, meta=false 2024-11-14T04:50:02,717 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 44289c305c03,38537,1731559799622 WAL splitting is done? wals=0, meta=false 2024-11-14T04:50:02,718 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 44289c305c03,38537,1731559799622 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,38537,1731559799622-splitting does not exist 2024-11-14T04:50:02,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN}] 2024-11-14T04:50:02,721 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN 2024-11-14T04:50:02,722 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T04:50:02,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:02,799 INFO [RS:1;44289c305c03:38537 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:50:02,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38537-0x100ef94af6b0002, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:02,799 INFO [RS:1;44289c305c03:38537 {}] regionserver.HRegionServer(1031): Exiting; stopping=44289c305c03,38537,1731559799622; zookeeper connection closed. 2024-11-14T04:50:02,800 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1d4dcaf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1d4dcaf 2024-11-14T04:50:02,873 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5e76b4f363a336ed8ed2087be33abe67, regionState=OPENING, regionLocation=44289c305c03,34263,1731559799574 2024-11-14T04:50:02,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN because future has completed 2024-11-14T04:50:02,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e76b4f363a336ed8ed2087be33abe67, server=44289c305c03,34263,1731559799574}] 2024-11-14T04:50:02,986 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/44289c305c03:0 server-side Connection retries=45 2024-11-14T04:50:02,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:50:02,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:50:02,986 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:50:02,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:50:02,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:50:02,986 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:50:02,986 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:50:02,987 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44805 2024-11-14T04:50:02,989 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44805 connecting to ZooKeeper ensemble=127.0.0.1:50033 2024-11-14T04:50:02,990 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:50:02,990 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:50:02,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448050x0, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:50:02,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44805-0x100ef94af6b0003 connected 2024-11-14T04:50:02,995 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-14T04:50:02,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-14T04:50:02,996 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:50:02,997 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.cache 2024-11-14T04:50:02,997 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T04:50:02,997 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T04:50:02,997 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T04:50:02,997 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T04:50:02,997 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T04:50:02,998 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:50:02,999 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-14T04:50:02,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T04:50:03,000 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-14T04:50:03,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:50:03,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44805 2024-11-14T04:50:03,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44805 2024-11-14T04:50:03,005 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-14T04:50:03,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44805 2024-11-14T04:50:03,005 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T04:50:03,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44805 2024-11-14T04:50:03,006 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-14T04:50:03,006 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence is complete. 2024-11-14T04:50:03,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44805 2024-11-14T04:50:03,009 INFO [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(746): ClusterId : 389c1722-e708-43f8-a99b-a2edafafcdd5 2024-11-14T04:50:03,009 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:50:03,013 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:50:03,013 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:50:03,015 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:50:03,016 DEBUG [RS:2;44289c305c03:44805 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4519e2bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=44289c305c03/172.17.0.2:0 2024-11-14T04:50:03,029 DEBUG [RS:2;44289c305c03:44805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;44289c305c03:44805 2024-11-14T04:50:03,029 INFO [RS:2;44289c305c03:44805 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:50:03,030 INFO [RS:2;44289c305c03:44805 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:50:03,030 DEBUG [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:50:03,031 INFO [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(2659): reportForDuty to master=44289c305c03,40261,1731559799511 with port=44805, startcode=1731559802985 2024-11-14T04:50:03,031 DEBUG [RS:2;44289c305c03:44805 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:50:03,032 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,033 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e76b4f363a336ed8ed2087be33abe67, NAME => 'testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:50:03,033 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731559800762 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,033 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56725, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:50:03,033 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:50:03,033 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,033 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,034 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40261 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 44289c305c03,44805,1731559802985 2024-11-14T04:50:03,034 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40261 {}] master.ServerManager(517): Registering regionserver=44289c305c03,44805,1731559802985 2024-11-14T04:50:03,034 INFO [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,035 INFO [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e76b4f363a336ed8ed2087be33abe67 columnFamilyName family 2024-11-14T04:50:03,035 DEBUG [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643 2024-11-14T04:50:03,035 DEBUG [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:50:03,035 DEBUG [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T04:50:03,036 DEBUG [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:50:03,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:50:03,038 DEBUG [StoreFileOpener-5e76b4f363a336ed8ed2087be33abe67-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a, delay=985 ms 2024-11-14T04:50:03,041 DEBUG [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a 2024-11-14T04:50:03,041 INFO [StoreOpener-5e76b4f363a336ed8ed2087be33abe67-1 {}] regionserver.HStore(327): Store=5e76b4f363a336ed8ed2087be33abe67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:50:03,042 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [44289c305c03,44805,1731559802985] 2024-11-14T04:50:03,042 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,042 DEBUG [RS:2;44289c305c03:44805 {}] zookeeper.ZKUtil(111): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/44289c305c03,44805,1731559802985 2024-11-14T04:50:03,043 WARN [RS:2;44289c305c03:44805 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:50:03,043 INFO [RS:2;44289c305c03:44805 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:50:03,043 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,043 DEBUG [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,44805,1731559802985 2024-11-14T04:50:03,043 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,044 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,044 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,046 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,046 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5e76b4f363a336ed8ed2087be33abe67; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73527501, jitterRate=0.09564514458179474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:50:03,046 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,048 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5e76b4f363a336ed8ed2087be33abe67: Running coprocessor pre-open hook at 1731559803034Writing region info on filesystem at 1731559803034Initializing all the Stores at 1731559803034Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731559803034Cleaning up temporary data from old regions at 1731559803044 (+10 ms)Running coprocessor post-open hooks at 1731559803047 (+3 ms)Region opened successfully at 1731559803048 (+1 ms) 2024-11-14T04:50:03,049 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67., pid=9, masterSystemTime=1731559803029 2024-11-14T04:50:03,051 INFO [RS:2;44289c305c03:44805 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:50:03,052 DEBUG [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,053 INFO [RS_OPEN_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,053 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5e76b4f363a336ed8ed2087be33abe67, regionState=OPEN, openSeqNum=10, regionLocation=44289c305c03,34263,1731559799574 2024-11-14T04:50:03,057 INFO [RS:2;44289c305c03:44805 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:50:03,058 INFO [RS:2;44289c305c03:44805 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:50:03,058 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,058 INFO [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:50:03,059 INFO [RS:2;44289c305c03:44805 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:50:03,059 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,059 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,059 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,059 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,059 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,059 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,059 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/44289c305c03:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/44289c305c03:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:50:03,060 DEBUG [RS:2;44289c305c03:44805 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/44289c305c03:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:50:03,061 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,061 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,061 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,061 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,061 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,061 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,44805,1731559802985-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:50:03,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e76b4f363a336ed8ed2087be33abe67, server=44289c305c03,34263,1731559799574 because future has completed 2024-11-14T04:50:03,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-14T04:50:03,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e76b4f363a336ed8ed2087be33abe67, server=44289c305c03,34263,1731559799574 in 188 msec 2024-11-14T04:50:03,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T04:50:03,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731559800762, region=5e76b4f363a336ed8ed2087be33abe67, ASSIGN in 347 msec 2024-11-14T04:50:03,069 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 44289c305c03,38537,1731559799622 after splitting done 2024-11-14T04:50:03,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 44289c305c03,38537,1731559799622, splitWal=true, meta=false in 367 msec 2024-11-14T04:50:03,088 INFO [RS:2;44289c305c03:44805 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:50:03,088 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=44289c305c03,44805,1731559802985-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,088 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,089 INFO [RS:2;44289c305c03:44805 {}] regionserver.Replication(171): 44289c305c03,44805,1731559802985 started 2024-11-14T04:50:03,108 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:50:03,108 INFO [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(1482): Serving as 44289c305c03,44805,1731559802985, RpcServer on 44289c305c03/172.17.0.2:44805, sessionid=0x100ef94af6b0003 2024-11-14T04:50:03,109 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:50:03,109 DEBUG [RS:2;44289c305c03:44805 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 44289c305c03,44805,1731559802985 2024-11-14T04:50:03,109 DEBUG [RS:2;44289c305c03:44805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,44805,1731559802985' 2024-11-14T04:50:03,109 DEBUG [RS:2;44289c305c03:44805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:50:03,110 DEBUG [RS:2;44289c305c03:44805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:50:03,111 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:50:03,111 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:50:03,111 DEBUG [RS:2;44289c305c03:44805 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 44289c305c03,44805,1731559802985 2024-11-14T04:50:03,111 DEBUG [RS:2;44289c305c03:44805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '44289c305c03,44805,1731559802985' 2024-11-14T04:50:03,111 DEBUG [RS:2;44289c305c03:44805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:50:03,111 DEBUG [RS:2;44289c305c03:44805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:50:03,112 DEBUG [RS:2;44289c305c03:44805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:50:03,112 INFO [RS:2;44289c305c03:44805 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:50:03,112 INFO [RS:2;44289c305c03:44805 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:50:03,213 INFO [RS:2;44289c305c03:44805 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:50:03,214 INFO [RS:2;44289c305c03:44805 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=44289c305c03%2C44805%2C1731559802985, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,44805,1731559802985, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs, maxLogs=55 2024-11-14T04:50:03,241 INFO [RS:2;44289c305c03:44805 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/WALs/44289c305c03,44805,1731559802985/44289c305c03%2C44805%2C1731559802985.1731559803215 2024-11-14T04:50:03,241 DEBUG [RS:2;44289c305c03:44805 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T04:50:03,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T04:50:03,609 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T04:50:03,609 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:50:03,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:03,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:03,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T04:50:03,610 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T04:50:03,610 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1613277258, stopped=false 2024-11-14T04:50:03,610 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=44289c305c03,40261,1731559799511 2024-11-14T04:50:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:50:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:50:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:50:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:50:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:50:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:50:03,612 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T04:50:03,612 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T04:50:03,612 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:50:03,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:03,613 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:50:03,613 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '44289c305c03,34263,1731559799574' ***** 2024-11-14T04:50:03,613 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T04:50:03,613 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:50:03,613 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '44289c305c03,44805,1731559802985' ***** 2024-11-14T04:50:03,613 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T04:50:03,613 INFO [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T04:50:03,613 INFO [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T04:50:03,613 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T04:50:03,613 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:50:03,613 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T04:50:03,614 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T04:50:03,614 INFO [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence 2024-11-14T04:50:03,614 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T04:50:03,614 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T04:50:03,614 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T04:50:03,614 INFO [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3b3985c-e4ec-9ecd-c31c-48d6486f0dc0/bucket.persistence 2024-11-14T04:50:03,614 INFO [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T04:50:03,614 INFO [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T04:50:03,614 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T04:50:03,614 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T04:50:03,615 INFO [regionserver/44289c305c03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:50:03,624 DEBUG [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1488): Thread RS:2;44289c305c03:44805 started persisting bucket cache to file 2024-11-14T04:50:03,624 DEBUG [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1488): Thread RS:0;44289c305c03:34263 started persisting bucket cache to file 2024-11-14T04:50:03,624 DEBUG [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:50:03,624 DEBUG [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T04:50:03,624 DEBUG [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:50:03,625 DEBUG [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T04:50:03,628 DEBUG [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T04:50:03,628 DEBUG [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731559803624 2024-11-14T04:50:03,628 DEBUG [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-14T04:50:03,628 DEBUG [RS:0;44289c305c03:34263 {}] bucket.BucketCache(1510): Thread RS:0;44289c305c03:34263 finished persisting bucket cache to file, renaming 2024-11-14T04:50:03,628 DEBUG [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731559803624 2024-11-14T04:50:03,628 DEBUG [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1510): Thread RS:2;44289c305c03:44805 finished persisting bucket cache to file, renaming 2024-11-14T04:50:03,628 WARN [RS:2;44289c305c03:44805 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-14T04:50:03,628 INFO [RS:0;44289c305c03:34263 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:50:03,629 INFO [RS:2;44289c305c03:44805 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:50:03,629 INFO [RS:0;44289c305c03:34263 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T04:50:03,629 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:50:03,629 INFO [RS:2;44289c305c03:44805 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T04:50:03,629 INFO [RS:0;44289c305c03:34263 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T04:50:03,629 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:50:03,629 INFO [RS:2;44289c305c03:44805 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T04:50:03,629 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,629 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(959): stopping server 44289c305c03,34263,1731559799574 2024-11-14T04:50:03,629 INFO [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(959): stopping server 44289c305c03,44805,1731559802985 2024-11-14T04:50:03,629 INFO [RS:0;44289c305c03:34263 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:50:03,629 INFO [RS:2;44289c305c03:44805 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:50:03,629 INFO [RS:0;44289c305c03:34263 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;44289c305c03:34263. 2024-11-14T04:50:03,629 INFO [RS:2;44289c305c03:44805 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;44289c305c03:44805. 2024-11-14T04:50:03,629 DEBUG [RS:0;44289c305c03:34263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:50:03,629 DEBUG [RS:2;44289c305c03:44805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:50:03,629 DEBUG [RS:0;44289c305c03:34263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:03,629 DEBUG [RS:2;44289c305c03:44805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:03,629 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T04:50:03,629 INFO [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(976): stopping server 44289c305c03,44805,1731559802985; all regions closed. 2024-11-14T04:50:03,630 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e76b4f363a336ed8ed2087be33abe67, disabling compactions & flushes 2024-11-14T04:50:03,630 INFO [RS:0;44289c305c03:34263 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:50:03,630 INFO [RS:0;44289c305c03:34263 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:50:03,630 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,630 INFO [RS:0;44289c305c03:34263 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:50:03,630 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,630 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T04:50:03,630 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. after waiting 0 ms 2024-11-14T04:50:03,630 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,630 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T04:50:03,630 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1325): Online Regions={5e76b4f363a336ed8ed2087be33abe67=testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T04:50:03,630 DEBUG [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5e76b4f363a336ed8ed2087be33abe67 2024-11-14T04:50:03,630 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T04:50:03,630 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T04:50:03,630 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T04:50:03,631 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T04:50:03,631 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T04:50:03,631 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T04:50:03,631 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-14T04:50:03,632 DEBUG [RS:2;44289c305c03:44805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs 2024-11-14T04:50:03,632 INFO [RS:2;44289c305c03:44805 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 44289c305c03%2C44805%2C1731559802985:(num 1731559803215) 2024-11-14T04:50:03,632 DEBUG [RS:2;44289c305c03:44805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:03,632 INFO [RS:2;44289c305c03:44805 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:50:03,632 INFO [RS:2;44289c305c03:44805 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:50:03,632 INFO [RS:2;44289c305c03:44805 {}] hbase.ChoreService(370): Chore service for: regionserver/44289c305c03:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T04:50:03,632 INFO [RS:2;44289c305c03:44805 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:50:03,632 INFO [RS:2;44289c305c03:44805 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:50:03,632 INFO [regionserver/44289c305c03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:50:03,633 INFO [RS:2;44289c305c03:44805 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:50:03,633 INFO [RS:2;44289c305c03:44805 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:50:03,633 INFO [RS:2;44289c305c03:44805 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44805 2024-11-14T04:50:03,633 DEBUG [StoreFileCloser-5e76b4f363a336ed8ed2087be33abe67-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/family/c93b926d4ebf47df9d74c26de41de93a 2024-11-14T04:50:03,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:50:03,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/44289c305c03,44805,1731559802985 2024-11-14T04:50:03,635 INFO [RS:2;44289c305c03:44805 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:50:03,636 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [44289c305c03,44805,1731559802985] 2024-11-14T04:50:03,638 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/44289c305c03,44805,1731559802985 already deleted, retry=false 2024-11-14T04:50:03,638 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 44289c305c03,44805,1731559802985 expired; onlineServers=1 2024-11-14T04:50:03,640 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/default/testBlockEvictionOnGracefulStop1731559800762/5e76b4f363a336ed8ed2087be33abe67/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-14T04:50:03,640 INFO [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,640 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e76b4f363a336ed8ed2087be33abe67: Waiting for close lock at 1731559803629Running coprocessor pre-close hooks at 1731559803629Disabling compacts and flushes for region at 1731559803629Disabling writes for close at 1731559803630 (+1 ms)Writing region close event to WAL at 1731559803634 (+4 ms)Running coprocessor post-close hooks at 1731559803640 (+6 ms)Closed at 1731559803640 2024-11-14T04:50:03,640 DEBUG [RS_CLOSE_REGION-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67. 2024-11-14T04:50:03,643 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a040c96c3cf64b838a6f9a24cf5fe3f4 2024-11-14T04:50:03,665 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/info/a040c96c3cf64b838a6f9a24cf5fe3f4 is 209, key is testBlockEvictionOnGracefulStop1731559800762,,1731559800766.5e76b4f363a336ed8ed2087be33abe67./info:regioninfo/1731559803053/Put/seqid=0 2024-11-14T04:50:03,665 INFO [regionserver/44289c305c03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:50:03,666 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/info/a040c96c3cf64b838a6f9a24cf5fe3f4 2024-11-14T04:50:03,678 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 67ec5ef9ee854664a82e0f1427e4bc3b 2024-11-14T04:50:03,700 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/ns/67ec5ef9ee854664a82e0f1427e4bc3b is 43, key is default/ns:d/1731559800641/Put/seqid=0 2024-11-14T04:50:03,701 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/ns/67ec5ef9ee854664a82e0f1427e4bc3b 2024-11-14T04:50:03,713 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 90d3c301d2334cb29493c1e8cd97930b 2024-11-14T04:50:03,729 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/table/90d3c301d2334cb29493c1e8cd97930b is 80, key is testBlockEvictionOnGracefulStop1731559800762/table:state/1731559801191/Put/seqid=0 2024-11-14T04:50:03,730 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/table/90d3c301d2334cb29493c1e8cd97930b 2024-11-14T04:50:03,733 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/info/a040c96c3cf64b838a6f9a24cf5fe3f4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/a040c96c3cf64b838a6f9a24cf5fe3f4 2024-11-14T04:50:03,735 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/a040c96c3cf64b838a6f9a24cf5fe3f4, delay=931 ms 2024-11-14T04:50:03,736 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/a040c96c3cf64b838a6f9a24cf5fe3f4, entries=14, sequenceid=13, filesize=7.5 K 2024-11-14T04:50:03,736 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/ns/67ec5ef9ee854664a82e0f1427e4bc3b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/67ec5ef9ee854664a82e0f1427e4bc3b 2024-11-14T04:50:03,737 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/67ec5ef9ee854664a82e0f1427e4bc3b, delay=990 ms 2024-11-14T04:50:03,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:03,737 INFO [RS:2;44289c305c03:44805 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:50:03,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44805-0x100ef94af6b0003, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:03,738 INFO [RS:2;44289c305c03:44805 {}] regionserver.HRegionServer(1031): Exiting; stopping=44289c305c03,44805,1731559802985; zookeeper connection closed. 2024-11-14T04:50:03,738 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4cd5e9fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4cd5e9fa 2024-11-14T04:50:03,738 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/67ec5ef9ee854664a82e0f1427e4bc3b, entries=2, sequenceid=13, filesize=5.0 K 2024-11-14T04:50:03,739 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/.tmp/table/90d3c301d2334cb29493c1e8cd97930b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/90d3c301d2334cb29493c1e8cd97930b 2024-11-14T04:50:03,740 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/90d3c301d2334cb29493c1e8cd97930b, delay=904 ms 2024-11-14T04:50:03,742 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/90d3c301d2334cb29493c1e8cd97930b, entries=2, sequenceid=13, filesize=5.3 K 2024-11-14T04:50:03,743 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=13, compaction requested=false 2024-11-14T04:50:03,744 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/info/a040c96c3cf64b838a6f9a24cf5fe3f4 2024-11-14T04:50:03,744 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/ns/67ec5ef9ee854664a82e0f1427e4bc3b 2024-11-14T04:50:03,745 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/table/90d3c301d2334cb29493c1e8cd97930b 2024-11-14T04:50:03,750 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-14T04:50:03,750 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T04:50:03,750 INFO [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T04:50:03,751 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731559803630Running coprocessor pre-close hooks at 1731559803630Disabling compacts and flushes for region at 1731559803630Disabling writes for close at 1731559803631 (+1 ms)Obtaining lock to block concurrent updates at 1731559803631Preparing flush snapshotting stores in 1588230740 at 1731559803631Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731559803631Flushing stores of hbase:meta,,1.1588230740 at 1731559803632 (+1 ms)Flushing 1588230740/info: creating writer at 1731559803632Flushing 1588230740/info: appending metadata at 1731559803664 (+32 ms)Flushing 1588230740/info: closing flushed file at 1731559803664Flushing 1588230740/ns: creating writer at 1731559803669 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731559803699 (+30 ms)Flushing 1588230740/ns: closing flushed file at 1731559803699Flushing 1588230740/table: creating writer at 1731559803704 (+5 ms)Flushing 1588230740/table: appending metadata at 1731559803728 (+24 ms)Flushing 1588230740/table: closing flushed file at 1731559803728Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ddde1e3: reopening flushed file at 1731559803732 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@657f78ba: reopening flushed file at 1731559803736 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66491fce: reopening flushed file at 1731559803739 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=13, compaction requested=false at 1731559803743 (+4 ms)Writing region close event to WAL at 1731559803745 (+2 ms)Running coprocessor post-close hooks at 1731559803750 (+5 ms)Closed at 1731559803750 2024-11-14T04:50:03,751 DEBUG [RS_CLOSE_META-regionserver/44289c305c03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T04:50:03,831 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(976): stopping server 44289c305c03,34263,1731559799574; all regions closed. 2024-11-14T04:50:03,833 DEBUG [RS:0;44289c305c03:34263 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs 2024-11-14T04:50:03,833 INFO [RS:0;44289c305c03:34263 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 44289c305c03%2C34263%2C1731559799574.meta:.meta(num 1731559800588) 2024-11-14T04:50:03,834 DEBUG [RS:0;44289c305c03:34263 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/oldWALs 2024-11-14T04:50:03,834 INFO [RS:0;44289c305c03:34263 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 44289c305c03%2C34263%2C1731559799574:(num 1731559800166) 2024-11-14T04:50:03,834 DEBUG [RS:0;44289c305c03:34263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:50:03,834 INFO [RS:0;44289c305c03:34263 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:50:03,834 INFO [RS:0;44289c305c03:34263 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:50:03,835 INFO [RS:0;44289c305c03:34263 {}] hbase.ChoreService(370): Chore service for: regionserver/44289c305c03:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T04:50:03,835 INFO [RS:0;44289c305c03:34263 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:50:03,835 INFO [regionserver/44289c305c03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:50:03,835 INFO [RS:0;44289c305c03:34263 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34263 2024-11-14T04:50:03,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:50:03,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/44289c305c03,34263,1731559799574 2024-11-14T04:50:03,837 INFO [RS:0;44289c305c03:34263 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:50:03,838 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [44289c305c03,34263,1731559799574] 2024-11-14T04:50:03,840 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/44289c305c03,34263,1731559799574 already deleted, retry=false 2024-11-14T04:50:03,840 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 44289c305c03,34263,1731559799574 expired; onlineServers=0 2024-11-14T04:50:03,840 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '44289c305c03,40261,1731559799511' ***** 2024-11-14T04:50:03,840 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T04:50:03,840 INFO [M:0;44289c305c03:40261 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:50:03,840 INFO [M:0;44289c305c03:40261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:50:03,840 DEBUG [M:0;44289c305c03:40261 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T04:50:03,840 DEBUG [M:0;44289c305c03:40261 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T04:50:03,840 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T04:50:03,840 DEBUG [master/44289c305c03:0:becomeActiveMaster-HFileCleaner.large.0-1731559799935 {}] cleaner.HFileCleaner(306): Exit Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.large.0-1731559799935,5,FailOnTimeoutGroup] 2024-11-14T04:50:03,840 DEBUG [master/44289c305c03:0:becomeActiveMaster-HFileCleaner.small.0-1731559799936 {}] cleaner.HFileCleaner(306): Exit Thread[master/44289c305c03:0:becomeActiveMaster-HFileCleaner.small.0-1731559799936,5,FailOnTimeoutGroup] 2024-11-14T04:50:03,841 INFO [M:0;44289c305c03:40261 {}] hbase.ChoreService(370): Chore service for: master/44289c305c03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T04:50:03,841 INFO [M:0;44289c305c03:40261 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:50:03,841 DEBUG [M:0;44289c305c03:40261 {}] master.HMaster(1795): Stopping service threads 2024-11-14T04:50:03,841 INFO [M:0;44289c305c03:40261 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T04:50:03,841 INFO [M:0;44289c305c03:40261 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T04:50:03,841 INFO [M:0;44289c305c03:40261 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T04:50:03,841 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T04:50:03,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T04:50:03,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:50:03,842 DEBUG [M:0;44289c305c03:40261 {}] zookeeper.ZKUtil(347): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T04:50:03,842 WARN [M:0;44289c305c03:40261 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T04:50:03,843 INFO [M:0;44289c305c03:40261 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/.lastflushedseqids 2024-11-14T04:50:03,850 INFO [M:0;44289c305c03:40261 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T04:50:03,850 INFO [M:0;44289c305c03:40261 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T04:50:03,850 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T04:50:03,850 INFO [M:0;44289c305c03:40261 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:50:03,850 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:50:03,850 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T04:50:03,850 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:50:03,850 INFO [M:0;44289c305c03:40261 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-14T04:50:03,860 DEBUG [M:0;44289c305c03:40261 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 673d5a3d30804af3ad5ce1dbc481fdcd 2024-11-14T04:50:03,874 DEBUG [M:0;44289c305c03:40261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/673d5a3d30804af3ad5ce1dbc481fdcd is 82, key is hbase:meta,,1/info:regioninfo/1731559800624/Put/seqid=0 2024-11-14T04:50:03,875 INFO [M:0;44289c305c03:40261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/673d5a3d30804af3ad5ce1dbc481fdcd 2024-11-14T04:50:03,883 DEBUG [M:0;44289c305c03:40261 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 29754d5127a94ed585421c3672303813 2024-11-14T04:50:03,898 DEBUG [M:0;44289c305c03:40261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/29754d5127a94ed585421c3672303813 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731559801196/Put/seqid=0 2024-11-14T04:50:03,899 INFO [M:0;44289c305c03:40261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/29754d5127a94ed585421c3672303813 2024-11-14T04:50:03,909 DEBUG [M:0;44289c305c03:40261 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7389f19ac3844f7f9d26521db0182040 2024-11-14T04:50:03,924 DEBUG [M:0;44289c305c03:40261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7389f19ac3844f7f9d26521db0182040 is 69, key is 44289c305c03,34263,1731559799574/rs:state/1731559799974/Put/seqid=0 2024-11-14T04:50:03,925 INFO [M:0;44289c305c03:40261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7389f19ac3844f7f9d26521db0182040 2024-11-14T04:50:03,927 INFO [M:0;44289c305c03:40261 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7389f19ac3844f7f9d26521db0182040 2024-11-14T04:50:03,927 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/673d5a3d30804af3ad5ce1dbc481fdcd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/673d5a3d30804af3ad5ce1dbc481fdcd 2024-11-14T04:50:03,929 INFO [M:0;44289c305c03:40261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/673d5a3d30804af3ad5ce1dbc481fdcd, entries=8, sequenceid=83, filesize=5.5 K 2024-11-14T04:50:03,930 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/29754d5127a94ed585421c3672303813 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/29754d5127a94ed585421c3672303813 2024-11-14T04:50:03,931 INFO [M:0;44289c305c03:40261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/29754d5127a94ed585421c3672303813, entries=9, sequenceid=83, filesize=6.4 K 2024-11-14T04:50:03,932 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7389f19ac3844f7f9d26521db0182040 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7389f19ac3844f7f9d26521db0182040 2024-11-14T04:50:03,933 INFO [M:0;44289c305c03:40261 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7389f19ac3844f7f9d26521db0182040 2024-11-14T04:50:03,933 INFO [M:0;44289c305c03:40261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6468b70-5af5-bbf6-5198-f88e6e5a95e0/8bc0ab32-0f42-95e9-56df-7c5b54453643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7389f19ac3844f7f9d26521db0182040, entries=3, sequenceid=83, filesize=5.3 K 2024-11-14T04:50:03,934 INFO [M:0;44289c305c03:40261 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=83, compaction requested=false 2024-11-14T04:50:03,936 INFO [M:0;44289c305c03:40261 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:50:03,936 DEBUG [M:0;44289c305c03:40261 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731559803850Disabling compacts and flushes for region at 1731559803850Disabling writes for close at 1731559803850Obtaining lock to block concurrent updates at 1731559803850Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731559803850Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1731559803851 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731559803851Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731559803851Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731559803873 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731559803874 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731559803876 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731559803898 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731559803898Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731559803901 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731559803924 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731559803924Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a7851f: reopening flushed file at 1731559803927 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64082413: reopening flushed file at 1731559803930 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b04d51b: reopening flushed file at 1731559803931 (+1 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=83, compaction requested=false at 1731559803934 (+3 ms)Writing region close event to WAL at 1731559803936 (+2 ms)Closed at 1731559803936 2024-11-14T04:50:03,937 INFO [M:0;44289c305c03:40261 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T04:50:03,937 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:50:03,937 INFO [M:0;44289c305c03:40261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40261 2024-11-14T04:50:03,938 INFO [M:0;44289c305c03:40261 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:50:03,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:03,939 INFO [RS:0;44289c305c03:34263 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:50:03,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34263-0x100ef94af6b0001, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:03,939 INFO [RS:0;44289c305c03:34263 {}] regionserver.HRegionServer(1031): Exiting; stopping=44289c305c03,34263,1731559799574; zookeeper connection closed. 2024-11-14T04:50:03,940 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@268ebc5f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@268ebc5f 2024-11-14T04:50:03,940 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T04:50:04,040 INFO [M:0;44289c305c03:40261 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:50:04,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:04,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40261-0x100ef94af6b0000, quorum=127.0.0.1:50033, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:50:04,044 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T04:50:04,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T04:50:04,046 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:50:04,055 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=737 (was 767), ProcessCount=11 (was 11), AvailableMemoryMB=8069 (was 8362)