2024-12-10 00:02:43,959 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 00:02:43,976 main DEBUG Took 0.010370 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 00:02:43,976 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 00:02:43,977 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 00:02:43,977 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 00:02:43,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:43,986 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 00:02:43,997 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:43,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:43,999 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:43,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,000 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,001 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,001 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,002 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,004 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,004 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,005 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:02:44,007 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,007 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 00:02:44,008 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:02:44,009 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 00:02:44,011 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 00:02:44,011 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 00:02:44,012 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 00:02:44,013 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 00:02:44,020 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 00:02:44,022 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 00:02:44,023 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 00:02:44,024 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 00:02:44,024 main DEBUG createAppenders(={Console}) 2024-12-10 00:02:44,024 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-10 00:02:44,025 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 00:02:44,025 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-10 00:02:44,025 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 00:02:44,026 main DEBUG OutputStream closed 2024-12-10 00:02:44,026 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 00:02:44,026 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 00:02:44,026 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-10 00:02:44,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 00:02:44,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 00:02:44,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 00:02:44,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 00:02:44,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 00:02:44,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 00:02:44,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 00:02:44,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 00:02:44,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 00:02:44,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 00:02:44,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 00:02:44,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 00:02:44,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 00:02:44,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 00:02:44,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 00:02:44,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 00:02:44,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 00:02:44,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 00:02:44,106 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 00:02:44,106 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-10 00:02:44,107 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 00:02:44,107 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-10T00:02:44,118 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-10 00:02:44,120 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 00:02:44,121 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T00:02:44,312 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa 2024-12-10T00:02:44,332 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=73, ProcessCount=11, AvailableMemoryMB=5070 2024-12-10T00:02:44,432 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T00:02:44,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:44,527 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/cluster_6141d48f-9da8-e7a3-cbde-9d17004819d1, deleteOnExit=true 2024-12-10T00:02:44,646 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/cluster_6141d48f-9da8-e7a3-cbde-9d17004819d1/zookeeper_0, clientPort=51910, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/cluster_6141d48f-9da8-e7a3-cbde-9d17004819d1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/cluster_6141d48f-9da8-e7a3-cbde-9d17004819d1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T00:02:44,656 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51910 2024-12-10T00:02:44,656 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:44,729 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c with version=8 2024-12-10T00:02:44,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/hbase-staging 2024-12-10T00:02:44,806 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T00:02:45,025 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2907d75fbb3e:0 server-side Connection retries=45 2024-12-10T00:02:45,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:02:45,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:02:45,177 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T00:02:45,251 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:02:45,284 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T00:02:45,294 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T00:02:45,297 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:02:45,334 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 112427 (auto-detected) 2024-12-10T00:02:45,336 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T00:02:45,357 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41847 2024-12-10T00:02:45,378 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41847 connecting to ZooKeeper ensemble=127.0.0.1:51910 2024-12-10T00:02:45,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418470x0, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:02:45,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41847-0x1000d15d5ac0000 connected 2024-12-10T00:02:45,580 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:45,582 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:45,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:45,594 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c, hbase.cluster.distributed=false 2024-12-10T00:02:45,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:02:45,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41847 2024-12-10T00:02:45,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41847 2024-12-10T00:02:45,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41847 2024-12-10T00:02:45,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41847 2024-12-10T00:02:45,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41847 2024-12-10T00:02:45,693 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2907d75fbb3e:0 server-side Connection retries=45 2024-12-10T00:02:45,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,695 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:02:45,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:02:45,697 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T00:02:45,699 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:02:45,700 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42235 2024-12-10T00:02:45,702 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42235 connecting to ZooKeeper ensemble=127.0.0.1:51910 2024-12-10T00:02:45,702 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:45,703 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:45,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422350x0, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:02:45,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422350x0, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:45,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42235-0x1000d15d5ac0001 connected 2024-12-10T00:02:45,735 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T00:02:45,749 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.cache 2024-12-10T00:02:45,756 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T00:02:45,758 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T00:02:45,759 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T00:02:45,761 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T00:02:45,761 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T00:02:45,767 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T00:02:45,768 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T00:02:45,768 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.persistence is complete. 2024-12-10T00:02:45,773 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T00:02:45,776 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T00:02:45,781 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:02:45,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42235 2024-12-10T00:02:45,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42235 2024-12-10T00:02:45,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42235 2024-12-10T00:02:45,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42235 2024-12-10T00:02:45,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42235 2024-12-10T00:02:45,804 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2907d75fbb3e:0 server-side Connection retries=45 2024-12-10T00:02:45,804 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,805 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:02:45,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:45,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:02:45,805 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T00:02:45,805 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:02:45,806 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43511 2024-12-10T00:02:45,809 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43511 connecting to ZooKeeper ensemble=127.0.0.1:51910 2024-12-10T00:02:45,809 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:45,810 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:45,834 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435110x0, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:02:45,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:435110x0, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:45,835 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43511-0x1000d15d5ac0002 connected 2024-12-10T00:02:45,835 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T00:02:45,836 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.cache 2024-12-10T00:02:45,836 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T00:02:45,837 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T00:02:45,837 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T00:02:45,837 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T00:02:45,837 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T00:02:45,837 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T00:02:45,838 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T00:02:45,838 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T00:02:45,838 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.persistence is complete. 2024-12-10T00:02:45,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T00:02:45,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:02:45,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43511 2024-12-10T00:02:45,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43511 2024-12-10T00:02:45,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43511 2024-12-10T00:02:45,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43511 2024-12-10T00:02:45,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43511 2024-12-10T00:02:45,874 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2907d75fbb3e:41847 2024-12-10T00:02:45,876 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:45,901 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:45,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:45,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:45,903 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:45,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T00:02:45,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:45,942 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T00:02:45,943 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:45,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:45,943 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T00:02:45,945 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2907d75fbb3e,41847,1733788964871 from backup master directory 2024-12-10T00:02:45,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:45,950 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:45,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:45,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:45,951 WARN [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:02:45,952 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:45,954 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T00:02:45,957 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T00:02:45,967 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/hbase.id] with ID: 162e8fb7-067b-464d-b33e-98564b61fd04 2024-12-10T00:02:45,967 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.tmp/hbase.id 2024-12-10T00:02:45,973 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/hbase.id] 2024-12-10T00:02:46,008 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:46,014 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T00:02:46,038 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-12-10T00:02:46,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,067 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,090 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:02:46,092 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T00:02:46,132 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:02:46,137 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:46,181 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store 2024-12-10T00:02:46,198 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T00:02:46,201 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:46,203 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T00:02:46,203 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:46,204 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:46,205 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T00:02:46,205 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:46,205 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:46,206 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733788966203Disabling compacts and flushes for region at 1733788966203Disabling writes for close at 1733788966205 (+2 ms)Writing region close event to WAL at 1733788966205Closed at 1733788966205 2024-12-10T00:02:46,206 WARN [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/.initializing 2024-12-10T00:02:46,207 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/WALs/2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:46,217 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:02:46,229 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2907d75fbb3e%2C41847%2C1733788964871, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/WALs/2907d75fbb3e,41847,1733788964871, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/oldWALs, maxLogs=10 2024-12-10T00:02:46,264 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/WALs/2907d75fbb3e,41847,1733788964871/2907d75fbb3e%2C41847%2C1733788964871.1733788966234 2024-12-10T00:02:46,265 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:46,265 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:46,266 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:46,269 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,270 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,312 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T00:02:46,342 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:46,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T00:02:46,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:46,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,355 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T00:02:46,355 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,356 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:46,356 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T00:02:46,361 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:46,362 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,367 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,367 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,371 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,372 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,374 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:02:46,377 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:46,382 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:02:46,383 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70010012, jitterRate=0.04323047399520874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:02:46,389 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733788966282Initializing all the Stores at 1733788966282Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788966283 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788966284 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788966284Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788966284Cleaning up temporary data from old regions at 1733788966372 (+88 ms)Region opened successfully at 1733788966389 (+17 ms) 2024-12-10T00:02:46,391 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T00:02:46,420 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb47360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-10T00:02:46,452 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T00:02:46,466 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T00:02:46,466 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T00:02:46,470 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T00:02:46,471 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-10T00:02:46,478 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-10T00:02:46,478 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T00:02:46,506 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T00:02:46,516 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T00:02:46,558 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T00:02:46,561 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T00:02:46,564 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T00:02:46,575 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T00:02:46,577 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T00:02:46,583 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T00:02:46,591 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T00:02:46,593 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T00:02:46,600 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T00:02:46,620 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T00:02:46,625 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T00:02:46,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:46,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:46,641 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:46,642 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,644 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2907d75fbb3e,41847,1733788964871, sessionid=0x1000d15d5ac0000, setting cluster-up flag (Was=false) 2024-12-10T00:02:46,666 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,692 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T00:02:46,695 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:46,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,717 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:46,741 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T00:02:46,743 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:46,749 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T00:02:46,758 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(746): ClusterId : 162e8fb7-067b-464d-b33e-98564b61fd04 2024-12-10T00:02:46,758 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(746): ClusterId : 162e8fb7-067b-464d-b33e-98564b61fd04 2024-12-10T00:02:46,760 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T00:02:46,760 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T00:02:46,768 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T00:02:46,768 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T00:02:46,768 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T00:02:46,768 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T00:02:46,776 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T00:02:46,776 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T00:02:46,777 DEBUG [RS:1;2907d75fbb3e:43511 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fcd514c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-10T00:02:46,777 DEBUG [RS:0;2907d75fbb3e:42235 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f9c1cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-10T00:02:46,791 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2907d75fbb3e:43511 2024-12-10T00:02:46,795 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T00:02:46,795 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T00:02:46,795 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T00:02:46,796 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2907d75fbb3e:42235 2024-12-10T00:02:46,796 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T00:02:46,796 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T00:02:46,796 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T00:02:46,798 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,41847,1733788964871 with port=42235, startcode=1733788965663 2024-12-10T00:02:46,798 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,41847,1733788964871 with port=43511, startcode=1733788965804 2024-12-10T00:02:46,809 DEBUG [RS:0;2907d75fbb3e:42235 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T00:02:46,809 DEBUG [RS:1;2907d75fbb3e:43511 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T00:02:46,827 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:46,837 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T00:02:46,846 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T00:02:46,853 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2907d75fbb3e,41847,1733788964871 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T00:02:46,860 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:46,860 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:46,860 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:46,861 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:46,861 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2907d75fbb3e:0, corePoolSize=10, maxPoolSize=10 2024-12-10T00:02:46,861 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:46,861 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:02:46,862 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:46,868 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:46,869 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T00:02:46,874 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733788996874 2024-12-10T00:02:46,875 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47463, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T00:02:46,875 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,875 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41701, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T00:02:46,875 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T00:02:46,876 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T00:02:46,877 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T00:02:46,881 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T00:02:46,881 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T00:02:46,882 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T00:02:46,882 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T00:02:46,883 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-10T00:02:46,883 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:46,890 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-10T00:02:46,891 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T00:02:46,893 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T00:02:46,893 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T00:02:46,898 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T00:02:46,899 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T00:02:46,901 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788966900,5,FailOnTimeoutGroup] 2024-12-10T00:02:46,902 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T00:02:46,902 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:46,903 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788966901,5,FailOnTimeoutGroup] 2024-12-10T00:02:46,903 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:46,903 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T00:02:46,905 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:46,905 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:46,916 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:46,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T00:02:46,919 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T00:02:46,919 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T00:02:46,919 WARN [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T00:02:46,919 WARN [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T00:02:46,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T00:02:46,922 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:46,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T00:02:46,928 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T00:02:46,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:46,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T00:02:46,934 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T00:02:46,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:46,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T00:02:46,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T00:02:46,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:46,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:46,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T00:02:46,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:46,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:46,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T00:02:46,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T00:02:46,943 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:02:46,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T00:02:46,949 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:02:46,950 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70927339, jitterRate=0.05689971148967743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:02:46,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733788966916Initializing all the Stores at 1733788966917 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788966917Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788966917Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788966917Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788966917Cleaning up temporary data from old regions at 1733788966942 (+25 ms)Region opened successfully at 1733788966952 (+10 ms) 2024-12-10T00:02:46,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T00:02:46,953 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T00:02:46,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T00:02:46,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T00:02:46,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T00:02:46,954 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:46,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733788966952Disabling compacts and flushes for region at 1733788966952Disabling writes for close at 1733788966953 (+1 ms)Writing region close event to WAL at 1733788966954 (+1 ms)Closed at 1733788966954 2024-12-10T00:02:46,956 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:46,957 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T00:02:46,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T00:02:46,970 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:02:46,972 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T00:02:47,021 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,41847,1733788964871 with port=42235, startcode=1733788965663 2024-12-10T00:02:47,021 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,41847,1733788964871 with port=43511, startcode=1733788965804 2024-12-10T00:02:47,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,025 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] master.ServerManager(517): Registering regionserver=2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,033 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:47,033 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] master.ServerManager(517): Registering regionserver=2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:47,034 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:47,034 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T00:02:47,034 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T00:02:47,036 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:47,036 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T00:02:47,036 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T00:02:47,123 WARN [2907d75fbb3e:41847 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T00:02:47,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:02:47,138 DEBUG [RS:1;2907d75fbb3e:43511 {}] zookeeper.ZKUtil(111): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,139 DEBUG [RS:0;2907d75fbb3e:42235 {}] zookeeper.ZKUtil(111): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:47,139 WARN [RS:1;2907d75fbb3e:43511 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:02:47,139 WARN [RS:0;2907d75fbb3e:42235 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:02:47,139 INFO [RS:1;2907d75fbb3e:43511 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:47,139 INFO [RS:0;2907d75fbb3e:42235 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:47,140 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:47,140 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,143 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2907d75fbb3e,43511,1733788965804] 2024-12-10T00:02:47,144 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2907d75fbb3e,42235,1733788965663] 2024-12-10T00:02:47,165 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T00:02:47,165 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T00:02:47,176 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T00:02:47,176 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T00:02:47,181 INFO [RS:0;2907d75fbb3e:42235 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T00:02:47,181 INFO [RS:1;2907d75fbb3e:43511 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T00:02:47,181 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,181 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,182 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T00:02:47,182 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T00:02:47,186 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T00:02:47,186 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T00:02:47,187 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,187 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,187 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,187 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,187 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,187 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,187 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,187 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,187 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,187 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:02:47,188 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,188 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,189 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:47,189 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:47,189 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:47,189 DEBUG [RS:0;2907d75fbb3e:42235 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:47,189 DEBUG [RS:1;2907d75fbb3e:43511 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:47,191 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,191 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,191 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,191 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,191 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,191 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,191 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,192 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,192 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,192 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,42235,1733788965663-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:02:47,192 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,192 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,43511,1733788965804-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:02:47,211 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T00:02:47,213 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,43511,1733788965804-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,213 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T00:02:47,213 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,213 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.Replication(171): 2907d75fbb3e,43511,1733788965804 started 2024-12-10T00:02:47,213 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,42235,1733788965663-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,213 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,213 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.Replication(171): 2907d75fbb3e,42235,1733788965663 started 2024-12-10T00:02:47,228 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,229 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1482): Serving as 2907d75fbb3e,43511,1733788965804, RpcServer on 2907d75fbb3e/172.17.0.2:43511, sessionid=0x1000d15d5ac0002 2024-12-10T00:02:47,229 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T00:02:47,230 DEBUG [RS:1;2907d75fbb3e:43511 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,230 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,43511,1733788965804' 2024-12-10T00:02:47,230 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T00:02:47,231 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T00:02:47,231 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T00:02:47,231 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T00:02:47,232 DEBUG [RS:1;2907d75fbb3e:43511 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,232 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,43511,1733788965804' 2024-12-10T00:02:47,232 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:47,232 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T00:02:47,232 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1482): Serving as 2907d75fbb3e,42235,1733788965663, RpcServer on 2907d75fbb3e/172.17.0.2:42235, sessionid=0x1000d15d5ac0001 2024-12-10T00:02:47,232 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T00:02:47,232 DEBUG [RS:0;2907d75fbb3e:42235 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:47,233 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,42235,1733788965663' 2024-12-10T00:02:47,233 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T00:02:47,233 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T00:02:47,234 DEBUG [RS:1;2907d75fbb3e:43511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T00:02:47,234 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T00:02:47,234 INFO [RS:1;2907d75fbb3e:43511 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T00:02:47,234 INFO [RS:1;2907d75fbb3e:43511 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T00:02:47,234 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T00:02:47,235 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T00:02:47,235 DEBUG [RS:0;2907d75fbb3e:42235 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:47,235 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,42235,1733788965663' 2024-12-10T00:02:47,235 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T00:02:47,236 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T00:02:47,236 DEBUG [RS:0;2907d75fbb3e:42235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T00:02:47,237 INFO [RS:0;2907d75fbb3e:42235 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T00:02:47,237 INFO [RS:0;2907d75fbb3e:42235 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T00:02:47,341 INFO [RS:1;2907d75fbb3e:43511 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:02:47,341 INFO [RS:0;2907d75fbb3e:42235 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:02:47,343 INFO [RS:1;2907d75fbb3e:43511 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C43511%2C1733788965804, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,43511,1733788965804, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:47,343 INFO [RS:0;2907d75fbb3e:42235 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C42235%2C1733788965663, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,42235,1733788965663, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:47,363 INFO [RS:0;2907d75fbb3e:42235 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,42235,1733788965663/2907d75fbb3e%2C42235%2C1733788965663.1733788967348 2024-12-10T00:02:47,363 INFO [RS:1;2907d75fbb3e:43511 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,43511,1733788965804/2907d75fbb3e%2C43511%2C1733788965804.1733788967348 2024-12-10T00:02:47,363 DEBUG [RS:0;2907d75fbb3e:42235 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:47,363 DEBUG [RS:1;2907d75fbb3e:43511 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:47,629 DEBUG [2907d75fbb3e:41847 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T00:02:47,643 DEBUG [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(204): Hosts are {2907d75fbb3e=0} racks are {/default-rack=0} 2024-12-10T00:02:47,645 DEBUG [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T00:02:47,645 DEBUG [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T00:02:47,646 DEBUG [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T00:02:47,646 DEBUG [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T00:02:47,646 INFO [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T00:02:47,646 INFO [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T00:02:47,646 DEBUG [2907d75fbb3e:41847 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T00:02:47,653 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,657 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,43511,1733788965804, state=OPENING 2024-12-10T00:02:47,716 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T00:02:47,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:47,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:47,725 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:47,727 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:47,727 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:47,727 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:47,728 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:02:47,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,43511,1733788965804}] 2024-12-10T00:02:47,909 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T00:02:47,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55203, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T00:02:47,919 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T00:02:47,919 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:47,920 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T00:02:47,921 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C43511%2C1733788965804.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,43511,1733788965804, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:47,936 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,43511,1733788965804/2907d75fbb3e%2C43511%2C1733788965804.meta.1733788967922.meta 2024-12-10T00:02:47,937 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:47,937 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:47,939 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T00:02:47,941 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T00:02:47,945 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T00:02:47,948 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T00:02:47,949 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:47,949 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T00:02:47,949 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T00:02:47,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T00:02:47,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T00:02:47,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:47,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:47,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T00:02:47,952 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T00:02:47,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:47,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:47,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T00:02:47,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T00:02:47,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:47,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:47,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T00:02:47,954 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T00:02:47,954 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:47,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:47,954 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T00:02:47,956 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:47,956 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:47,957 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T00:02:47,957 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T00:02:47,958 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:02:47,959 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T00:02:47,960 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68617225, jitterRate=0.022476330399513245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:02:47,960 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T00:02:47,962 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733788967949Writing region info on filesystem at 1733788967950 (+1 ms)Initializing all the Stores at 1733788967950Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788967950Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788967950Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788967950Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788967950Cleaning up temporary data from old regions at 1733788967957 (+7 ms)Running coprocessor post-open hooks at 1733788967961 (+4 ms)Region opened successfully at 1733788967962 (+1 ms) 2024-12-10T00:02:47,968 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733788967902 2024-12-10T00:02:47,976 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T00:02:47,976 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T00:02:47,977 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:47,978 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,43511,1733788965804, state=OPEN 2024-12-10T00:02:48,008 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:48,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:48,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:48,008 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:48,008 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:48,008 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:48,009 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:48,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T00:02:48,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,43511,1733788965804 in 279 msec 2024-12-10T00:02:48,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T00:02:48,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0530 sec 2024-12-10T00:02:48,019 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:48,019 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T00:02:48,040 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:02:48,041 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,43511,1733788965804, seqNum=-1] 2024-12-10T00:02:48,058 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:02:48,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52673, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:02:48,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2930 sec 2024-12-10T00:02:48,077 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733788968077, completionTime=-1 2024-12-10T00:02:48,080 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T00:02:48,080 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T00:02:48,103 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T00:02:48,103 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733789028103 2024-12-10T00:02:48,103 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733789088103 2024-12-10T00:02:48,103 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-10T00:02:48,105 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T00:02:48,111 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,41847,1733788964871-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:48,111 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,41847,1733788964871-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:48,112 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,41847,1733788964871-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:48,113 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2907d75fbb3e:41847, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:48,113 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:48,114 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:48,120 DEBUG [master/2907d75fbb3e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T00:02:48,137 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.185sec 2024-12-10T00:02:48,138 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T00:02:48,140 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T00:02:48,141 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T00:02:48,141 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T00:02:48,141 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T00:02:48,142 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,41847,1733788964871-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:02:48,142 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,41847,1733788964871-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T00:02:48,144 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T00:02:48,145 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T00:02:48,145 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,41847,1733788964871-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:48,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:02:48,169 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T00:02:48,169 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T00:02:48,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,41847,-1 for getting cluster id 2024-12-10T00:02:48,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:02:48,182 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '162e8fb7-067b-464d-b33e-98564b61fd04' 2024-12-10T00:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:02:48,185 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "162e8fb7-067b-464d-b33e-98564b61fd04" 2024-12-10T00:02:48,186 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59eaab00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:02:48,186 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,41847,-1] 2024-12-10T00:02:48,188 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:02:48,190 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:48,191 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:02:48,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:02:48,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:02:48,201 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,43511,1733788965804, seqNum=-1] 2024-12-10T00:02:48,202 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:02:48,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:02:48,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:48,236 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:02:48,241 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:48,243 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77ed1971 2024-12-10T00:02:48,244 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:02:48,246 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:02:48,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733788968230', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:02:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 2024-12-10T00:02:48,258 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:02:48,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733788968230" procId is: 4 2024-12-10T00:02:48,261 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:48,262 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:02:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:48,273 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 365dc3f8e44d1a05676dfebeb058522f, NAME => 'testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733788968230', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:48,279 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:48,279 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(1722): Closing 365dc3f8e44d1a05676dfebeb058522f, disabling compactions & flushes 2024-12-10T00:02:48,279 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:48,279 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:48,279 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. after waiting 0 ms 2024-12-10T00:02:48,279 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:48,279 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:48,279 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733788968230-pool-0 {}] regionserver.HRegion(1676): Region close journal for 365dc3f8e44d1a05676dfebeb058522f: Waiting for close lock at 1733788968279Disabling compacts and flushes for region at 1733788968279Disabling writes for close at 1733788968279Writing region close event to WAL at 1733788968279Closed at 1733788968279 2024-12-10T00:02:48,281 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:02:48,284 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733788968281"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788968281"}]},"ts":"1733788968281"} 2024-12-10T00:02:48,288 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:02:48,290 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:02:48,292 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733788968230","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788968290"}]},"ts":"1733788968290"} 2024-12-10T00:02:48,296 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733788968230, state=ENABLING in hbase:meta 2024-12-10T00:02:48,296 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {2907d75fbb3e=0} racks are {/default-rack=0} 2024-12-10T00:02:48,297 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T00:02:48,297 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T00:02:48,297 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T00:02:48,297 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T00:02:48,297 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T00:02:48,297 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T00:02:48,297 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T00:02:48,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, ASSIGN}] 2024-12-10T00:02:48,300 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, ASSIGN 2024-12-10T00:02:48,301 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, ASSIGN; state=OFFLINE, location=2907d75fbb3e,43511,1733788965804; forceNewPlan=false, retain=false 2024-12-10T00:02:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:48,453 INFO [2907d75fbb3e:41847 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T00:02:48,454 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=365dc3f8e44d1a05676dfebeb058522f, regionState=OPENING, regionLocation=2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:48,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, ASSIGN because future has completed 2024-12-10T00:02:48,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,43511,1733788965804}] 2024-12-10T00:02:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:48,616 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:48,617 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 365dc3f8e44d1a05676dfebeb058522f, NAME => 'testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:48,618 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733788968230 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,618 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:48,618 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,618 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,619 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,625 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 365dc3f8e44d1a05676dfebeb058522f columnFamilyName family 2024-12-10T00:02:48,625 DEBUG [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:48,626 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.HStore(327): Store=365dc3f8e44d1a05676dfebeb058522f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:48,626 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,627 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,628 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,628 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,628 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,629 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,635 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:02:48,635 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 365dc3f8e44d1a05676dfebeb058522f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71890791, jitterRate=0.07125626504421234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T00:02:48,635 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:48,636 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 365dc3f8e44d1a05676dfebeb058522f: Running coprocessor pre-open hook at 1733788968618Writing region info on filesystem at 1733788968618Initializing all the Stores at 1733788968619 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788968619Cleaning up temporary data from old regions at 1733788968628 (+9 ms)Running coprocessor post-open hooks at 1733788968635 (+7 ms)Region opened successfully at 1733788968636 (+1 ms) 2024-12-10T00:02:48,637 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f., pid=6, masterSystemTime=1733788968611 2024-12-10T00:02:48,640 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:48,640 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:48,641 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=365dc3f8e44d1a05676dfebeb058522f, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:48,643 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2907d75fbb3e,43511,1733788965804, table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-10T00:02:48,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,43511,1733788965804 because future has completed 2024-12-10T00:02:48,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T00:02:48,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,43511,1733788965804 in 187 msec 2024-12-10T00:02:48,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T00:02:48,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, ASSIGN in 348 msec 2024-12-10T00:02:48,650 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:02:48,650 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733788968230","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788968650"}]},"ts":"1733788968650"} 2024-12-10T00:02:48,652 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733788968230, state=ENABLED in hbase:meta 2024-12-10T00:02:48,653 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:02:48,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733788968230 in 400 msec 2024-12-10T00:02:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:48,902 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733788968230 completed 2024-12-10T00:02:48,902 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733788968230 get assigned. Timeout = 60000ms 2024-12-10T00:02:48,908 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T00:02:48,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733788968230 assigned to meta. Checking AM states. 2024-12-10T00:02:48,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T00:02:48,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733788968230 assigned. 2024-12-10T00:02:48,924 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733788968230', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f., hostname=2907d75fbb3e,43511,1733788965804, seqNum=2] 2024-12-10T00:02:48,934 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 365dc3f8e44d1a05676dfebeb058522f 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T00:02:48,955 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:48,985 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/.tmp/family/89ef9b09b0d046b8839643d321a918ac is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T00:02:48,989 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/.tmp/family/89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:49,017 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/.tmp/family/89ef9b09b0d046b8839643d321a918ac as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:49,018 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac, delay=964 ms 2024-12-10T00:02:49,019 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T00:02:49,025 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 365dc3f8e44d1a05676dfebeb058522f in 89ms, sequenceid=6, compaction requested=false 2024-12-10T00:02:49,026 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733788968230' 2024-12-10T00:02:49,028 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 365dc3f8e44d1a05676dfebeb058522f: 2024-12-10T00:02:49,987 DEBUG [hfile-prefetch-1733788969018 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 89ef9b09b0d046b8839643d321a918ac_0. 2024-12-10T00:02:49,995 DEBUG [hfile-prefetch-1733788969018 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 89ef9b09b0d046b8839643d321a918ac_117. 2024-12-10T00:02:49,996 DEBUG [hfile-prefetch-1733788969018 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac, with total blocks 2, and data blocks 1 2024-12-10T00:02:49,996 DEBUG [hfile-prefetch-1733788969018 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T00:02:49,996 DEBUG [hfile-prefetch-1733788969018 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T00:02:49,997 DEBUG [hfile-prefetch-1733788969018 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac. Will sleep 100ms and try the verification again. 2024-12-10T00:02:50,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=365dc3f8e44d1a05676dfebeb058522f, source=2907d75fbb3e,43511,1733788965804, destination=2907d75fbb3e,42235,1733788965663, warming up region on 2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T00:02:50,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=365dc3f8e44d1a05676dfebeb058522f, source=2907d75fbb3e,43511,1733788965804, destination=2907d75fbb3e,42235,1733788965663, running balancer 2024-12-10T00:02:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, REOPEN/MOVE 2024-12-10T00:02:50,049 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, REOPEN/MOVE 2024-12-10T00:02:50,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39693, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T00:02:50,050 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=365dc3f8e44d1a05676dfebeb058522f, regionState=CLOSING, regionLocation=2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:50,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(7855): Warmup {ENCODED => 365dc3f8e44d1a05676dfebeb058522f, NAME => 'testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:50,051 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,052 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 365dc3f8e44d1a05676dfebeb058522f columnFamilyName family 2024-12-10T00:02:50,052 DEBUG [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:50,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, REOPEN/MOVE because future has completed 2024-12-10T00:02:50,054 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-10T00:02:50,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,43511,1733788965804}] 2024-12-10T00:02:50,055 DEBUG [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:50,055 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.HStore(327): Store=365dc3f8e44d1a05676dfebeb058522f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(1722): Closing 365dc3f8e44d1a05676dfebeb058522f, disabling compactions & flushes 2024-12-10T00:02:50,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. after waiting 0 ms 2024-12-10T00:02:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,057 DEBUG [StoreFileCloser-365dc3f8e44d1a05676dfebeb058522f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:50,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42235 {}] regionserver.HRegion(1676): Region close journal for 365dc3f8e44d1a05676dfebeb058522f: Waiting for close lock at 1733788970056Disabling compacts and flushes for region at 1733788970056Disabling writes for close at 1733788970056Writing region close event to WAL at 1733788970058 (+2 ms)Closed at 1733788970058 2024-12-10T00:02:50,059 WARN [hfile-prefetch-1733788969018 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-10T00:02:50,060 DEBUG [hfile-prefetch-1733788969018 {}] hfile.PrefetchExecutor(121): Prefetch completed for 89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-10T00:02:50,141 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T00:02:50,141 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T00:02:50,142 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:50,142 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:50,162 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-10T00:02:50,162 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733788970141 2024-12-10T00:02:50,162 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T00:02:50,212 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,213 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-10T00:02:50,214 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 365dc3f8e44d1a05676dfebeb058522f, disabling compactions & flushes 2024-12-10T00:02:50,214 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,214 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,214 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. after waiting 0 ms 2024-12-10T00:02:50,214 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,218 DEBUG [StoreFileCloser-365dc3f8e44d1a05676dfebeb058522f-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 89ef9b09b0d046b8839643d321a918ac, starting offset: 0, end offset: 9223372036854775807 2024-12-10T00:02:50,221 DEBUG [StoreFileCloser-365dc3f8e44d1a05676dfebeb058522f-family-1 {}] bucket.BucketCache(856): removed key 89ef9b09b0d046b8839643d321a918ac_0 from back map with offset lock 2094080 in the evict process 2024-12-10T00:02:50,225 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T00:02:50,226 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,227 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 365dc3f8e44d1a05676dfebeb058522f: Waiting for close lock at 1733788970214Running coprocessor pre-close hooks at 1733788970214Disabling compacts and flushes for region at 1733788970214Disabling writes for close at 1733788970214Writing region close event to WAL at 1733788970221 (+7 ms)Running coprocessor post-close hooks at 1733788970226 (+5 ms)Closed at 1733788970226 2024-12-10T00:02:50,227 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 365dc3f8e44d1a05676dfebeb058522f move to 2907d75fbb3e,42235,1733788965663 record at close sequenceid=6 2024-12-10T00:02:50,230 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,230 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=365dc3f8e44d1a05676dfebeb058522f, regionState=CLOSED 2024-12-10T00:02:50,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,43511,1733788965804 because future has completed 2024-12-10T00:02:50,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T00:02:50,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,43511,1733788965804 in 180 msec 2024-12-10T00:02:50,236 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, REOPEN/MOVE; state=CLOSED, location=2907d75fbb3e,42235,1733788965663; forceNewPlan=false, retain=false 2024-12-10T00:02:50,263 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T00:02:50,263 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T00:02:50,263 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:50,263 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:50,264 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T00:02:50,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T00:02:50,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:50,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:50,268 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T00:02:50,268 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733788970263 2024-12-10T00:02:50,268 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T00:02:50,269 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T00:02:50,269 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733788970264 2024-12-10T00:02:50,269 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T00:02:50,387 INFO [2907d75fbb3e:41847 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T00:02:50,387 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=365dc3f8e44d1a05676dfebeb058522f, regionState=OPENING, regionLocation=2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:50,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, REOPEN/MOVE because future has completed 2024-12-10T00:02:50,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,42235,1733788965663}] 2024-12-10T00:02:50,550 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,551 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 365dc3f8e44d1a05676dfebeb058522f, NAME => 'testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:50,552 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733788968230 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,553 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:50,553 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,553 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,555 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,557 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 365dc3f8e44d1a05676dfebeb058522f columnFamilyName family 2024-12-10T00:02:50,557 DEBUG [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:50,559 DEBUG [StoreFileOpener-365dc3f8e44d1a05676dfebeb058522f-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac, delay=953 ms 2024-12-10T00:02:50,561 DEBUG [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:50,561 INFO [StoreOpener-365dc3f8e44d1a05676dfebeb058522f-1 {}] regionserver.HStore(327): Store=365dc3f8e44d1a05676dfebeb058522f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:50,561 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,562 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,562 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,562 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,562 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,566 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,567 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 365dc3f8e44d1a05676dfebeb058522f; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60923019, jitterRate=-0.09217627346515656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T00:02:50,567 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,568 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 365dc3f8e44d1a05676dfebeb058522f: Running coprocessor pre-open hook at 1733788970554Writing region info on filesystem at 1733788970554Initializing all the Stores at 1733788970554Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788970554Cleaning up temporary data from old regions at 1733788970562 (+8 ms)Running coprocessor post-open hooks at 1733788970567 (+5 ms)Region opened successfully at 1733788970567 2024-12-10T00:02:50,568 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f., pid=9, masterSystemTime=1733788970545 2024-12-10T00:02:50,571 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,571 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=365dc3f8e44d1a05676dfebeb058522f, regionState=OPEN, openSeqNum=10, regionLocation=2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:50,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,42235,1733788965663 because future has completed 2024-12-10T00:02:50,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-10T00:02:50,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 365dc3f8e44d1a05676dfebeb058522f, server=2907d75fbb3e,42235,1733788965663 in 184 msec 2024-12-10T00:02:50,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733788968230, region=365dc3f8e44d1a05676dfebeb058522f, REOPEN/MOVE in 531 msec 2024-12-10T00:02:50,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T00:02:50,602 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:02:50,602 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:50,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:50,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:50,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:02:50,607 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T00:02:50,608 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-12-10T00:02:50,608 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2907d75fbb3e,41847,1733788964871 2024-12-10T00:02:50,624 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:50,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:50,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:50,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:50,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:50,625 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:50,625 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T00:02:50,625 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:02:50,626 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:50,626 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:50,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:50,626 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:50,626 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:50,627 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2907d75fbb3e,42235,1733788965663' ***** 2024-12-10T00:02:50,627 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T00:02:50,627 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2907d75fbb3e,43511,1733788965804' ***** 2024-12-10T00:02:50,627 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T00:02:50,627 INFO [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T00:02:50,628 INFO [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T00:02:50,628 INFO [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.persistence 2024-12-10T00:02:50,628 INFO [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/bucket.persistence 2024-12-10T00:02:50,628 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T00:02:50,628 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T00:02:50,628 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T00:02:50,628 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T00:02:50,628 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T00:02:50,628 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T00:02:50,629 INFO [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T00:02:50,629 INFO [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T00:02:50,629 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T00:02:50,629 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T00:02:50,639 DEBUG [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1487): Thread RS:0;2907d75fbb3e:42235 started persisting bucket cache to file 2024-12-10T00:02:50,639 DEBUG [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1487): Thread RS:1;2907d75fbb3e:43511 started persisting bucket cache to file 2024-12-10T00:02:50,639 DEBUG [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:50,639 DEBUG [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:50,639 DEBUG [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:50,639 DEBUG [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:50,641 DEBUG [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T00:02:50,641 DEBUG [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T00:02:50,641 DEBUG [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733788970639 2024-12-10T00:02:50,641 DEBUG [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733788970639 2024-12-10T00:02:50,641 DEBUG [RS:1;2907d75fbb3e:43511 {}] bucket.BucketCache(1509): Thread RS:1;2907d75fbb3e:43511 finished persisting bucket cache to file, renaming 2024-12-10T00:02:50,642 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T00:02:50,642 INFO [RS:1;2907d75fbb3e:43511 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T00:02:50,642 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T00:02:50,642 INFO [RS:1;2907d75fbb3e:43511 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T00:02:50,642 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(959): stopping server 2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:50,642 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:02:50,642 INFO [RS:1;2907d75fbb3e:43511 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2907d75fbb3e:43511. 2024-12-10T00:02:50,642 DEBUG [RS:1;2907d75fbb3e:43511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:50,642 DEBUG [RS:1;2907d75fbb3e:43511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:50,643 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T00:02:50,643 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T00:02:50,643 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T00:02:50,643 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T00:02:50,643 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T00:02:50,643 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T00:02:50,644 DEBUG [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:02:50,644 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T00:02:50,644 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T00:02:50,644 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T00:02:50,644 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T00:02:50,644 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T00:02:50,644 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T00:02:50,644 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-10T00:02:50,653 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 80a8dc3768324c97bd9c0d5a5ce66da4 2024-12-10T00:02:50,658 DEBUG [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1509): Thread RS:0;2907d75fbb3e:42235 finished persisting bucket cache to file, renaming 2024-12-10T00:02:50,658 WARN [RS:0;2907d75fbb3e:42235 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T00:02:50,658 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T00:02:50,658 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T00:02:50,658 INFO [RS:0;2907d75fbb3e:42235 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T00:02:50,659 INFO [RS:0;2907d75fbb3e:42235 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T00:02:50,659 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(3091): Received CLOSE for 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,659 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(959): stopping server 2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:50,659 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:02:50,659 INFO [RS:0;2907d75fbb3e:42235 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2907d75fbb3e:42235. 2024-12-10T00:02:50,659 DEBUG [RS:0;2907d75fbb3e:42235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:50,659 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T00:02:50,659 DEBUG [RS:0;2907d75fbb3e:42235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:50,659 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 365dc3f8e44d1a05676dfebeb058522f, disabling compactions & flushes 2024-12-10T00:02:50,659 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T00:02:50,659 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,660 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1325): Online Regions={365dc3f8e44d1a05676dfebeb058522f=testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f.} 2024-12-10T00:02:50,660 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,660 DEBUG [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1351): Waiting on 365dc3f8e44d1a05676dfebeb058522f 2024-12-10T00:02:50,660 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. after waiting 0 ms 2024-12-10T00:02:50,660 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,660 DEBUG [StoreFileCloser-365dc3f8e44d1a05676dfebeb058522f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/family/89ef9b09b0d046b8839643d321a918ac 2024-12-10T00:02:50,665 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnRegionMove1733788968230/365dc3f8e44d1a05676dfebeb058522f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-10T00:02:50,665 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,665 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 365dc3f8e44d1a05676dfebeb058522f: Waiting for close lock at 1733788970659Running coprocessor pre-close hooks at 1733788970659Disabling compacts and flushes for region at 1733788970659Disabling writes for close at 1733788970660 (+1 ms)Writing region close event to WAL at 1733788970661 (+1 ms)Running coprocessor post-close hooks at 1733788970665 (+4 ms)Closed at 1733788970665 2024-12-10T00:02:50,665 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f. 2024-12-10T00:02:50,670 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/80a8dc3768324c97bd9c0d5a5ce66da4 is 205, key is testBlockEvictionOnRegionMove1733788968230,,1733788968247.365dc3f8e44d1a05676dfebeb058522f./info:regioninfo/1733788970571/Put/seqid=0 2024-12-10T00:02:50,671 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/80a8dc3768324c97bd9c0d5a5ce66da4 2024-12-10T00:02:50,679 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 132b281eba434d5ca02317642bec2f1e 2024-12-10T00:02:50,690 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/ns/132b281eba434d5ca02317642bec2f1e is 43, key is default/ns:d/1733788968065/Put/seqid=0 2024-12-10T00:02:50,691 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/ns/132b281eba434d5ca02317642bec2f1e 2024-12-10T00:02:50,697 INFO [regionserver/2907d75fbb3e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:50,697 INFO [regionserver/2907d75fbb3e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:50,699 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 43bd00e550d8476fb213e961e68d7f47 2024-12-10T00:02:50,710 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/table/43bd00e550d8476fb213e961e68d7f47 is 78, key is testBlockEvictionOnRegionMove1733788968230/table:state/1733788968650/Put/seqid=0 2024-12-10T00:02:50,710 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/table/43bd00e550d8476fb213e961e68d7f47 2024-12-10T00:02:50,712 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/80a8dc3768324c97bd9c0d5a5ce66da4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/80a8dc3768324c97bd9c0d5a5ce66da4 2024-12-10T00:02:50,713 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/80a8dc3768324c97bd9c0d5a5ce66da4, delay=917 ms 2024-12-10T00:02:50,714 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/80a8dc3768324c97bd9c0d5a5ce66da4, entries=15, sequenceid=15, filesize=7.6 K 2024-12-10T00:02:50,715 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/ns/132b281eba434d5ca02317642bec2f1e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/132b281eba434d5ca02317642bec2f1e 2024-12-10T00:02:50,716 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/132b281eba434d5ca02317642bec2f1e, delay=903 ms 2024-12-10T00:02:50,717 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/132b281eba434d5ca02317642bec2f1e, entries=2, sequenceid=15, filesize=5.0 K 2024-12-10T00:02:50,717 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/table/43bd00e550d8476fb213e961e68d7f47 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/43bd00e550d8476fb213e961e68d7f47 2024-12-10T00:02:50,718 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/43bd00e550d8476fb213e961e68d7f47, delay=961 ms 2024-12-10T00:02:50,719 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/43bd00e550d8476fb213e961e68d7f47, entries=2, sequenceid=15, filesize=5.3 K 2024-12-10T00:02:50,720 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=15, compaction requested=false 2024-12-10T00:02:50,720 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T00:02:50,721 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/80a8dc3768324c97bd9c0d5a5ce66da4 2024-12-10T00:02:50,722 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/132b281eba434d5ca02317642bec2f1e 2024-12-10T00:02:50,723 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/43bd00e550d8476fb213e961e68d7f47 2024-12-10T00:02:50,726 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-10T00:02:50,727 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T00:02:50,727 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:50,727 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733788970643Running coprocessor pre-close hooks at 1733788970644 (+1 ms)Disabling compacts and flushes for region at 1733788970644Disabling writes for close at 1733788970644Obtaining lock to block concurrent updates at 1733788970644Preparing flush snapshotting stores in 1588230740 at 1733788970644Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733788970645 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733788970645Flushing 1588230740/info: creating writer at 1733788970645Flushing 1588230740/info: appending metadata at 1733788970669 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733788970669Flushing 1588230740/ns: creating writer at 1733788970674 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733788970690 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733788970690Flushing 1588230740/table: creating writer at 1733788970693 (+3 ms)Flushing 1588230740/table: appending metadata at 1733788970709 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733788970709Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@278c2e7d: reopening flushed file at 1733788970712 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53d54fcc: reopening flushed file at 1733788970714 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a460a3: reopening flushed file at 1733788970717 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=15, compaction requested=false at 1733788970720 (+3 ms)Writing region close event to WAL at 1733788970723 (+3 ms)Running coprocessor post-close hooks at 1733788970727 (+4 ms)Closed at 1733788970727 2024-12-10T00:02:50,728 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:50,844 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(976): stopping server 2907d75fbb3e,43511,1733788965804; all regions closed. 2024-12-10T00:02:50,853 DEBUG [RS:1;2907d75fbb3e:43511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:50,853 INFO [RS:1;2907d75fbb3e:43511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C43511%2C1733788965804.meta:.meta(num 1733788967922) 2024-12-10T00:02:50,858 DEBUG [RS:1;2907d75fbb3e:43511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:50,858 INFO [RS:1;2907d75fbb3e:43511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C43511%2C1733788965804:(num 1733788967348) 2024-12-10T00:02:50,858 DEBUG [RS:1;2907d75fbb3e:43511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:50,858 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:50,858 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:02:50,859 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.ChoreService(370): Chore service for: regionserver/2907d75fbb3e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T00:02:50,859 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:02:50,859 INFO [regionserver/2907d75fbb3e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:02:50,859 INFO [RS:1;2907d75fbb3e:43511 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43511 2024-12-10T00:02:50,860 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(976): stopping server 2907d75fbb3e,42235,1733788965663; all regions closed. 2024-12-10T00:02:50,862 DEBUG [RS:0;2907d75fbb3e:42235 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:50,862 INFO [RS:0;2907d75fbb3e:42235 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C42235%2C1733788965663:(num 1733788967348) 2024-12-10T00:02:50,862 DEBUG [RS:0;2907d75fbb3e:42235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:50,862 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:50,862 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:02:50,863 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.ChoreService(370): Chore service for: regionserver/2907d75fbb3e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T00:02:50,863 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T00:02:50,863 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T00:02:50,863 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T00:02:50,863 INFO [regionserver/2907d75fbb3e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:02:50,863 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:02:50,863 INFO [RS:0;2907d75fbb3e:42235 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42235 2024-12-10T00:02:50,890 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2907d75fbb3e,43511,1733788965804 2024-12-10T00:02:50,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:02:50,891 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:02:50,899 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:02:50,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2907d75fbb3e,42235,1733788965663 2024-12-10T00:02:50,899 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2907d75fbb3e,43511,1733788965804] 2024-12-10T00:02:50,899 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f5af4477b20@10ffd2f7 rejected from java.util.concurrent.ThreadPoolExecutor@55fd4e8c[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-10T00:02:50,916 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2907d75fbb3e,43511,1733788965804 already deleted, retry=false 2024-12-10T00:02:50,916 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2907d75fbb3e,43511,1733788965804 expired; onlineServers=1 2024-12-10T00:02:50,916 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2907d75fbb3e,42235,1733788965663] 2024-12-10T00:02:50,924 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2907d75fbb3e,42235,1733788965663 already deleted, retry=false 2024-12-10T00:02:50,924 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2907d75fbb3e,42235,1733788965663 expired; onlineServers=0 2024-12-10T00:02:50,924 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2907d75fbb3e,41847,1733788964871' ***** 2024-12-10T00:02:50,924 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T00:02:50,924 INFO [M:0;2907d75fbb3e:41847 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:02:50,924 INFO [M:0;2907d75fbb3e:41847 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:02:50,924 DEBUG [M:0;2907d75fbb3e:41847 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T00:02:50,925 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T00:02:50,925 DEBUG [M:0;2907d75fbb3e:41847 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T00:02:50,925 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788966900 {}] cleaner.HFileCleaner(306): Exit Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788966900,5,FailOnTimeoutGroup] 2024-12-10T00:02:50,925 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788966901 {}] cleaner.HFileCleaner(306): Exit Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788966901,5,FailOnTimeoutGroup] 2024-12-10T00:02:50,925 INFO [M:0;2907d75fbb3e:41847 {}] hbase.ChoreService(370): Chore service for: master/2907d75fbb3e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T00:02:50,925 INFO [M:0;2907d75fbb3e:41847 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:02:50,926 DEBUG [M:0;2907d75fbb3e:41847 {}] master.HMaster(1795): Stopping service threads 2024-12-10T00:02:50,926 INFO [M:0;2907d75fbb3e:41847 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T00:02:50,926 INFO [M:0;2907d75fbb3e:41847 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T00:02:50,926 INFO [M:0;2907d75fbb3e:41847 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T00:02:50,927 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T00:02:50,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T00:02:50,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:50,933 DEBUG [M:0;2907d75fbb3e:41847 {}] zookeeper.ZKUtil(347): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T00:02:50,933 WARN [M:0;2907d75fbb3e:41847 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T00:02:50,933 INFO [M:0;2907d75fbb3e:41847 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.lastflushedseqids 2024-12-10T00:02:50,942 INFO [M:0;2907d75fbb3e:41847 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T00:02:50,942 INFO [M:0;2907d75fbb3e:41847 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T00:02:50,942 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T00:02:50,942 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:50,942 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:50,943 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T00:02:50,943 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:50,943 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-10T00:02:50,951 DEBUG [M:0;2907d75fbb3e:41847 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2c7f85b8e1d444b3817c414461e091ca 2024-12-10T00:02:50,962 DEBUG [M:0;2907d75fbb3e:41847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c7f85b8e1d444b3817c414461e091ca is 82, key is hbase:meta,,1/info:regioninfo/1733788967977/Put/seqid=0 2024-12-10T00:02:50,963 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c7f85b8e1d444b3817c414461e091ca 2024-12-10T00:02:50,970 DEBUG [M:0;2907d75fbb3e:41847 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 457fd9ae226944669cc82049b18441dd 2024-12-10T00:02:50,981 DEBUG [M:0;2907d75fbb3e:41847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/457fd9ae226944669cc82049b18441dd is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733788968655/Put/seqid=0 2024-12-10T00:02:50,981 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/457fd9ae226944669cc82049b18441dd 2024-12-10T00:02:50,988 DEBUG [M:0;2907d75fbb3e:41847 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7a1edcd65b0f43cabe7c62c43bf69021 2024-12-10T00:02:50,998 DEBUG [M:0;2907d75fbb3e:41847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a1edcd65b0f43cabe7c62c43bf69021 is 69, key is 2907d75fbb3e,42235,1733788965663/rs:state/1733788967034/Put/seqid=0 2024-12-10T00:02:50,999 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a1edcd65b0f43cabe7c62c43bf69021 2024-12-10T00:02:51,000 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c7f85b8e1d444b3817c414461e091ca as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c7f85b8e1d444b3817c414461e091ca 2024-12-10T00:02:51,002 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c7f85b8e1d444b3817c414461e091ca, entries=8, sequenceid=84, filesize=5.5 K 2024-12-10T00:02:51,002 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/457fd9ae226944669cc82049b18441dd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/457fd9ae226944669cc82049b18441dd 2024-12-10T00:02:51,003 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/457fd9ae226944669cc82049b18441dd, entries=9, sequenceid=84, filesize=6.5 K 2024-12-10T00:02:51,004 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a1edcd65b0f43cabe7c62c43bf69021 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a1edcd65b0f43cabe7c62c43bf69021 2024-12-10T00:02:51,005 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a1edcd65b0f43cabe7c62c43bf69021, entries=2, sequenceid=84, filesize=5.1 K 2024-12-10T00:02:51,006 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 62ms, sequenceid=84, compaction requested=false 2024-12-10T00:02:51,007 INFO [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:51,007 DEBUG [M:0;2907d75fbb3e:41847 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733788970942Disabling compacts and flushes for region at 1733788970942Disabling writes for close at 1733788970943 (+1 ms)Obtaining lock to block concurrent updates at 1733788970943Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733788970943Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733788970943Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733788970944 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733788970944Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733788970962 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733788970962Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733788970965 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733788970980 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733788970980Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733788970983 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733788970998 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733788970998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1733788971000 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1733788971002 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1733788971003 (+1 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 62ms, sequenceid=84, compaction requested=false at 1733788971006 (+3 ms)Writing region close event to WAL at 1733788971007 (+1 ms)Closed at 1733788971007 2024-12-10T00:02:51,008 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:51,008 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43511-0x1000d15d5ac0002, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:51,008 INFO [RS:1;2907d75fbb3e:43511 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:02:51,008 INFO [RS:1;2907d75fbb3e:43511 {}] regionserver.HRegionServer(1031): Exiting; stopping=2907d75fbb3e,43511,1733788965804; zookeeper connection closed. 2024-12-10T00:02:51,008 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@61df5b93 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@61df5b93 2024-12-10T00:02:51,009 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:02:51,009 INFO [M:0;2907d75fbb3e:41847 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T00:02:51,009 INFO [M:0;2907d75fbb3e:41847 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41847 2024-12-10T00:02:51,009 INFO [M:0;2907d75fbb3e:41847 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:02:51,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:51,016 INFO [RS:0;2907d75fbb3e:42235 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:02:51,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42235-0x1000d15d5ac0001, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:51,016 INFO [RS:0;2907d75fbb3e:42235 {}] regionserver.HRegionServer(1031): Exiting; stopping=2907d75fbb3e,42235,1733788965663; zookeeper connection closed. 2024-12-10T00:02:51,016 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5b1ae9e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5b1ae9e7 2024-12-10T00:02:51,017 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-10T00:02:51,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:51,133 INFO [M:0;2907d75fbb3e:41847 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:02:51,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1000d15d5ac0000, quorum=127.0.0.1:51910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:51,141 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T00:02:51,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T00:02:51,160 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,167 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2907d75fbb3e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1733788969018 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733788970713 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2907d75fbb3e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1733788970559 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2907d75fbb3e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733788970716 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2907d75fbb3e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=69 (was 73), ProcessCount=11 (was 11), AvailableMemoryMB=4706 (was 5070) 2024-12-10T00:02:51,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=69, ProcessCount=11, AvailableMemoryMB=4706 2024-12-10T00:02:51,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/hadoop-log-dir so I do NOT create it in target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703 2024-12-10T00:02:51,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/hadoop-tmp-dir so I do NOT create it in target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703 2024-12-10T00:02:51,173 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,175 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/cluster_5cdada53-766a-0ee7-79de-afb215df26be, deleteOnExit=true 2024-12-10T00:02:51,178 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/cluster_5cdada53-766a-0ee7-79de-afb215df26be/zookeeper_0, clientPort=60691, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/cluster_5cdada53-766a-0ee7-79de-afb215df26be/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/cluster_5cdada53-766a-0ee7-79de-afb215df26be/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T00:02:51,179 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60691 2024-12-10T00:02:51,192 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c with version=8 2024-12-10T00:02:51,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/hbase-staging 2024-12-10T00:02:51,195 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2907d75fbb3e:0 server-side Connection retries=45 2024-12-10T00:02:51,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,195 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:02:51,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:02:51,195 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T00:02:51,196 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:02:51,196 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40255 2024-12-10T00:02:51,198 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40255 connecting to ZooKeeper ensemble=127.0.0.1:60691 2024-12-10T00:02:51,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402550x0, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:02:51,261 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40255-0x1000d15ef460000 connected 2024-12-10T00:02:51,335 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,336 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,338 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:51,339 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c, hbase.cluster.distributed=false 2024-12-10T00:02:51,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:02:51,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40255 2024-12-10T00:02:51,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40255 2024-12-10T00:02:51,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40255 2024-12-10T00:02:51,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40255 2024-12-10T00:02:51,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40255 2024-12-10T00:02:51,364 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2907d75fbb3e:0 server-side Connection retries=45 2024-12-10T00:02:51,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,364 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:02:51,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:02:51,364 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T00:02:51,365 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:02:51,366 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34505 2024-12-10T00:02:51,368 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34505 connecting to ZooKeeper ensemble=127.0.0.1:60691 2024-12-10T00:02:51,368 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,368 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:345050x0, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:02:51,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:51,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34505-0x1000d15ef460001 connected 2024-12-10T00:02:51,377 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T00:02:51,377 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.cache 2024-12-10T00:02:51,377 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T00:02:51,378 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T00:02:51,378 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T00:02:51,378 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T00:02:51,378 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T00:02:51,378 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T00:02:51,378 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T00:02:51,378 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T00:02:51,379 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence is complete. 2024-12-10T00:02:51,379 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T00:02:51,380 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:02:51,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34505 2024-12-10T00:02:51,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34505 2024-12-10T00:02:51,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34505 2024-12-10T00:02:51,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34505 2024-12-10T00:02:51,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34505 2024-12-10T00:02:51,397 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2907d75fbb3e:0 server-side Connection retries=45 2024-12-10T00:02:51,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,398 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:02:51,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:51,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:02:51,398 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T00:02:51,398 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:02:51,399 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36597 2024-12-10T00:02:51,400 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36597 connecting to ZooKeeper ensemble=127.0.0.1:60691 2024-12-10T00:02:51,400 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,401 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365970x0, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:02:51,410 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36597-0x1000d15ef460002 connected 2024-12-10T00:02:51,410 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:51,410 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T00:02:51,411 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.cache 2024-12-10T00:02:51,411 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T00:02:51,411 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T00:02:51,411 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T00:02:51,411 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T00:02:51,412 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T00:02:51,412 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T00:02:51,412 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T00:02:51,412 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence is complete. 2024-12-10T00:02:51,412 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T00:02:51,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T00:02:51,414 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:02:51,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36597 2024-12-10T00:02:51,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36597 2024-12-10T00:02:51,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36597 2024-12-10T00:02:51,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36597 2024-12-10T00:02:51,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36597 2024-12-10T00:02:51,427 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2907d75fbb3e:40255 2024-12-10T00:02:51,427 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:51,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:51,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:51,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:51,435 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:51,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T00:02:51,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T00:02:51,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,443 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T00:02:51,444 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2907d75fbb3e,40255,1733788971194 from backup master directory 2024-12-10T00:02:51,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:51,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:51,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:51,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:02:51,451 WARN [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:02:51,451 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:51,453 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/hbase.id] with ID: c8cdf554-9edf-496d-87f6-2944d756a9d6 2024-12-10T00:02:51,453 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.tmp/hbase.id 2024-12-10T00:02:51,458 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/hbase.id] 2024-12-10T00:02:51,496 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:51,496 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T00:02:51,499 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-12-10T00:02:51,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,513 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:02:51,515 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T00:02:51,515 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:51,538 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store 2024-12-10T00:02:51,546 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:51,546 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T00:02:51,546 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:51,546 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:51,546 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T00:02:51,546 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:51,546 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:51,546 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733788971546Disabling compacts and flushes for region at 1733788971546Disabling writes for close at 1733788971546Writing region close event to WAL at 1733788971546Closed at 1733788971546 2024-12-10T00:02:51,547 WARN [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/.initializing 2024-12-10T00:02:51,547 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/WALs/2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:51,554 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:02:51,558 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2907d75fbb3e%2C40255%2C1733788971194, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/WALs/2907d75fbb3e,40255,1733788971194, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/oldWALs, maxLogs=10 2024-12-10T00:02:51,575 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/WALs/2907d75fbb3e,40255,1733788971194/2907d75fbb3e%2C40255%2C1733788971194.1733788971558 2024-12-10T00:02:51,575 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:51,575 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:51,575 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:51,576 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,576 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T00:02:51,580 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:51,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T00:02:51,584 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:51,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T00:02:51,587 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:51,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T00:02:51,591 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:51,591 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,592 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,592 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,592 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,592 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,592 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:02:51,593 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:02:51,598 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:02:51,598 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60279175, jitterRate=-0.1017702966928482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:02:51,599 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733788971576Initializing all the Stores at 1733788971576Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788971576Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788971576Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788971576Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788971576Cleaning up temporary data from old regions at 1733788971592 (+16 ms)Region opened successfully at 1733788971598 (+6 ms) 2024-12-10T00:02:51,599 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T00:02:51,602 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4534fe5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-10T00:02:51,603 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T00:02:51,603 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T00:02:51,603 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T00:02:51,603 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T00:02:51,604 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-10T00:02:51,604 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-10T00:02:51,604 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T00:02:51,606 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T00:02:51,606 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T00:02:51,634 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T00:02:51,635 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T00:02:51,636 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T00:02:51,642 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T00:02:51,643 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T00:02:51,644 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T00:02:51,651 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T00:02:51,652 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T00:02:51,659 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T00:02:51,662 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T00:02:51,667 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T00:02:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,677 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2907d75fbb3e,40255,1733788971194, sessionid=0x1000d15ef460000, setting cluster-up flag (Was=false) 2024-12-10T00:02:51,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,717 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T00:02:51,720 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:51,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:51,759 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T00:02:51,761 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:51,761 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T00:02:51,763 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:51,763 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T00:02:51,763 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T00:02:51,763 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2907d75fbb3e,40255,1733788971194 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T00:02:51,765 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:51,765 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:51,765 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:51,765 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:02:51,766 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2907d75fbb3e:0, corePoolSize=10, maxPoolSize=10 2024-12-10T00:02:51,766 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,766 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:02:51,766 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,768 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733789001768 2024-12-10T00:02:51,768 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:51,768 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T00:02:51,768 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T00:02:51,768 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T00:02:51,768 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T00:02:51,768 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T00:02:51,768 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T00:02:51,769 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T00:02:51,769 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,769 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,769 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T00:02:51,769 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T00:02:51,769 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T00:02:51,770 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T00:02:51,771 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T00:02:51,771 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T00:02:51,771 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788971771,5,FailOnTimeoutGroup] 2024-12-10T00:02:51,771 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788971771,5,FailOnTimeoutGroup] 2024-12-10T00:02:51,771 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,771 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T00:02:51,771 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,772 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,783 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T00:02:51,783 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:51,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:51,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T00:02:51,791 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T00:02:51,791 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:51,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T00:02:51,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T00:02:51,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:51,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T00:02:51,798 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T00:02:51,798 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:51,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T00:02:51,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T00:02:51,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:51,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:51,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T00:02:51,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:51,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:51,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T00:02:51,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T00:02:51,803 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:02:51,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T00:02:51,808 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:02:51,809 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68834806, jitterRate=0.02571853995323181}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:02:51,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733788971788Initializing all the Stores at 1733788971788Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788971788Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788971789 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788971789Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788971789Cleaning up temporary data from old regions at 1733788971803 (+14 ms)Region opened successfully at 1733788971810 (+7 ms) 2024-12-10T00:02:51,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T00:02:51,810 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T00:02:51,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T00:02:51,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T00:02:51,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T00:02:51,811 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:51,811 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733788971810Disabling compacts and flushes for region at 1733788971810Disabling writes for close at 1733788971810Writing region close event to WAL at 1733788971811 (+1 ms)Closed at 1733788971811 2024-12-10T00:02:51,812 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:51,812 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T00:02:51,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T00:02:51,814 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:02:51,815 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T00:02:51,820 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(746): ClusterId : c8cdf554-9edf-496d-87f6-2944d756a9d6 2024-12-10T00:02:51,820 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(746): ClusterId : c8cdf554-9edf-496d-87f6-2944d756a9d6 2024-12-10T00:02:51,820 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T00:02:51,820 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T00:02:51,852 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T00:02:51,852 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T00:02:51,852 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T00:02:51,852 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T00:02:51,861 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T00:02:51,861 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T00:02:51,861 DEBUG [RS:0;2907d75fbb3e:34505 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d31d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-10T00:02:51,862 DEBUG [RS:1;2907d75fbb3e:36597 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40141449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-10T00:02:51,875 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2907d75fbb3e:34505 2024-12-10T00:02:51,875 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2907d75fbb3e:36597 2024-12-10T00:02:51,875 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T00:02:51,875 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T00:02:51,875 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T00:02:51,875 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T00:02:51,875 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T00:02:51,875 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T00:02:51,876 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,40255,1733788971194 with port=34505, startcode=1733788971363 2024-12-10T00:02:51,876 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,40255,1733788971194 with port=36597, startcode=1733788971397 2024-12-10T00:02:51,876 DEBUG [RS:0;2907d75fbb3e:34505 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T00:02:51,876 DEBUG [RS:1;2907d75fbb3e:36597 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T00:02:51,878 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44189, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T00:02:51,878 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46805, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T00:02:51,879 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40255 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:51,879 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40255 {}] master.ServerManager(517): Registering regionserver=2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:51,880 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40255 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:51,880 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40255 {}] master.ServerManager(517): Registering regionserver=2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:51,880 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:51,880 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T00:02:51,880 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T00:02:51,881 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:51,881 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T00:02:51,881 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T00:02:51,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:02:51,922 DEBUG [RS:1;2907d75fbb3e:36597 {}] zookeeper.ZKUtil(111): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:51,922 WARN [RS:1;2907d75fbb3e:36597 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:02:51,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2907d75fbb3e,34505,1733788971363] 2024-12-10T00:02:51,923 INFO [RS:1;2907d75fbb3e:36597 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:51,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2907d75fbb3e,36597,1733788971397] 2024-12-10T00:02:51,923 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:51,923 DEBUG [RS:0;2907d75fbb3e:34505 {}] zookeeper.ZKUtil(111): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:51,923 WARN [RS:0;2907d75fbb3e:34505 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:02:51,924 INFO [RS:0;2907d75fbb3e:34505 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:51,924 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:51,931 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T00:02:51,935 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T00:02:51,935 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T00:02:51,936 INFO [RS:1;2907d75fbb3e:36597 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T00:02:51,936 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,936 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T00:02:51,936 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T00:02:51,937 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T00:02:51,937 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,937 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,937 INFO [RS:0;2907d75fbb3e:34505 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T00:02:51,937 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,937 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,937 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,937 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T00:02:51,937 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,937 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:02:51,938 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,938 DEBUG [RS:1;2907d75fbb3e:36597 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:51,939 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,939 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,939 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:51,939 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:51,939 DEBUG [RS:0;2907d75fbb3e:34505 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:51,939 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,939 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,939 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,939 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,939 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,939 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,36597,1733788971397-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:02:51,940 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,940 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,940 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,940 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,940 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,940 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,34505,1733788971363-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:02:51,952 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T00:02:51,953 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,36597,1733788971397-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,953 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,953 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.Replication(171): 2907d75fbb3e,36597,1733788971397 started 2024-12-10T00:02:51,962 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T00:02:51,962 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,34505,1733788971363-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,963 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,963 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.Replication(171): 2907d75fbb3e,34505,1733788971363 started 2024-12-10T00:02:51,966 WARN [2907d75fbb3e:40255 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T00:02:51,966 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,966 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1482): Serving as 2907d75fbb3e,36597,1733788971397, RpcServer on 2907d75fbb3e/172.17.0.2:36597, sessionid=0x1000d15ef460002 2024-12-10T00:02:51,966 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T00:02:51,966 DEBUG [RS:1;2907d75fbb3e:36597 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:51,966 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,36597,1733788971397' 2024-12-10T00:02:51,966 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T00:02:51,967 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T00:02:51,968 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T00:02:51,968 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T00:02:51,968 DEBUG [RS:1;2907d75fbb3e:36597 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:51,968 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,36597,1733788971397' 2024-12-10T00:02:51,968 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T00:02:51,969 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T00:02:51,969 DEBUG [RS:1;2907d75fbb3e:36597 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T00:02:51,969 INFO [RS:1;2907d75fbb3e:36597 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T00:02:51,969 INFO [RS:1;2907d75fbb3e:36597 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T00:02:51,980 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:51,980 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1482): Serving as 2907d75fbb3e,34505,1733788971363, RpcServer on 2907d75fbb3e/172.17.0.2:34505, sessionid=0x1000d15ef460001 2024-12-10T00:02:51,981 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T00:02:51,981 DEBUG [RS:0;2907d75fbb3e:34505 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:51,981 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,34505,1733788971363' 2024-12-10T00:02:51,981 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T00:02:51,981 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T00:02:51,982 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T00:02:51,982 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T00:02:51,982 DEBUG [RS:0;2907d75fbb3e:34505 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:51,982 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,34505,1733788971363' 2024-12-10T00:02:51,982 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T00:02:51,983 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T00:02:51,983 DEBUG [RS:0;2907d75fbb3e:34505 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T00:02:51,983 INFO [RS:0;2907d75fbb3e:34505 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T00:02:51,983 INFO [RS:0;2907d75fbb3e:34505 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T00:02:52,070 INFO [RS:1;2907d75fbb3e:36597 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:02:52,071 INFO [RS:1;2907d75fbb3e:36597 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C36597%2C1733788971397, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:52,084 INFO [RS:0;2907d75fbb3e:34505 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:02:52,085 INFO [RS:0;2907d75fbb3e:34505 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C34505%2C1733788971363, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,34505,1733788971363, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:52,091 INFO [RS:1;2907d75fbb3e:36597 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397/2907d75fbb3e%2C36597%2C1733788971397.1733788972073 2024-12-10T00:02:52,092 DEBUG [RS:1;2907d75fbb3e:36597 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:52,103 INFO [RS:0;2907d75fbb3e:34505 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,34505,1733788971363/2907d75fbb3e%2C34505%2C1733788971363.1733788972087 2024-12-10T00:02:52,103 DEBUG [RS:0;2907d75fbb3e:34505 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:52,216 DEBUG [2907d75fbb3e:40255 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T00:02:52,217 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(204): Hosts are {2907d75fbb3e=0} racks are {/default-rack=0} 2024-12-10T00:02:52,218 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T00:02:52,218 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T00:02:52,218 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T00:02:52,218 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T00:02:52,218 INFO [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T00:02:52,218 INFO [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T00:02:52,218 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T00:02:52,219 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:52,221 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,36597,1733788971397, state=OPENING 2024-12-10T00:02:52,251 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T00:02:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:52,260 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:02:52,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:52,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:52,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:52,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,36597,1733788971397}] 2024-12-10T00:02:52,413 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T00:02:52,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T00:02:52,421 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T00:02:52,422 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:52,422 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T00:02:52,423 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C36597%2C1733788971397.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:52,440 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397/2907d75fbb3e%2C36597%2C1733788971397.meta.1733788972424.meta 2024-12-10T00:02:52,440 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:52,441 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:52,441 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T00:02:52,441 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T00:02:52,441 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T00:02:52,441 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T00:02:52,441 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:52,441 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T00:02:52,442 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T00:02:52,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T00:02:52,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T00:02:52,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:52,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:52,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T00:02:52,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T00:02:52,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:52,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:52,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T00:02:52,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T00:02:52,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:52,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:52,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T00:02:52,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T00:02:52,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:52,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:52,445 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T00:02:52,446 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:52,446 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:52,446 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T00:02:52,446 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T00:02:52,447 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:02:52,447 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T00:02:52,448 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73383403, jitterRate=0.09349791705608368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:02:52,448 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T00:02:52,448 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733788972442Writing region info on filesystem at 1733788972442Initializing all the Stores at 1733788972442Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788972442Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788972442Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788972442Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788972442Cleaning up temporary data from old regions at 1733788972446 (+4 ms)Running coprocessor post-open hooks at 1733788972448 (+2 ms)Region opened successfully at 1733788972448 2024-12-10T00:02:52,449 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733788972413 2024-12-10T00:02:52,452 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T00:02:52,452 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T00:02:52,453 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:52,454 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,36597,1733788971397, state=OPEN 2024-12-10T00:02:52,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:52,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:52,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:52,484 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:52,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:52,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:52,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:52,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T00:02:52,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,36597,1733788971397 in 224 msec 2024-12-10T00:02:52,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T00:02:52,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 674 msec 2024-12-10T00:02:52,490 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:02:52,490 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T00:02:52,493 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:02:52,493 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,36597,1733788971397, seqNum=-1] 2024-12-10T00:02:52,493 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:02:52,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33069, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:02:52,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 738 msec 2024-12-10T00:02:52,500 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733788972500, completionTime=-1 2024-12-10T00:02:52,500 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T00:02:52,500 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T00:02:52,503 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T00:02:52,503 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733789032503 2024-12-10T00:02:52,503 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733789092503 2024-12-10T00:02:52,503 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-10T00:02:52,503 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40255,1733788971194-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:52,503 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40255,1733788971194-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:52,503 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40255,1733788971194-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:52,504 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2907d75fbb3e:40255, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:52,504 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:52,504 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:52,508 DEBUG [master/2907d75fbb3e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T00:02:52,509 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.058sec 2024-12-10T00:02:52,509 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T00:02:52,509 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T00:02:52,509 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T00:02:52,509 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T00:02:52,509 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T00:02:52,510 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40255,1733788971194-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:02:52,510 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40255,1733788971194-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T00:02:52,510 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T00:02:52,510 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T00:02:52,511 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40255,1733788971194-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:52,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:02:52,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,40255,-1 for getting cluster id 2024-12-10T00:02:52,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:02:52,522 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c8cdf554-9edf-496d-87f6-2944d756a9d6' 2024-12-10T00:02:52,522 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:02:52,522 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c8cdf554-9edf-496d-87f6-2944d756a9d6" 2024-12-10T00:02:52,523 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4977ef25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:02:52,523 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,40255,-1] 2024-12-10T00:02:52,523 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:02:52,524 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:52,525 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:02:52,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:02:52,527 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:02:52,528 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,36597,1733788971397, seqNum=-1] 2024-12-10T00:02:52,529 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:02:52,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:02:52,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:52,534 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:02:52,535 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:52,535 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7285d9b4 2024-12-10T00:02:52,535 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:02:52,537 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:02:52,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733788972533', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:02:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40255 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 2024-12-10T00:02:52,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:02:52,539 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:52,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733788972533" procId is: 4 2024-12-10T00:02:52,540 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:02:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:52,549 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d9c786f3802c6e255c34a64099197346, NAME => 'testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733788972533', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:52,554 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:52,555 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(1722): Closing d9c786f3802c6e255c34a64099197346, disabling compactions & flushes 2024-12-10T00:02:52,555 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:52,555 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:52,555 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. after waiting 0 ms 2024-12-10T00:02:52,555 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:52,555 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:52,555 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733788972533-pool-0 {}] regionserver.HRegion(1676): Region close journal for d9c786f3802c6e255c34a64099197346: Waiting for close lock at 1733788972555Disabling compacts and flushes for region at 1733788972555Disabling writes for close at 1733788972555Writing region close event to WAL at 1733788972555Closed at 1733788972555 2024-12-10T00:02:52,556 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:02:52,557 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733788972556"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788972556"}]},"ts":"1733788972556"} 2024-12-10T00:02:52,559 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:02:52,560 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:02:52,560 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733788972533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788972560"}]},"ts":"1733788972560"} 2024-12-10T00:02:52,563 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733788972533, state=ENABLING in hbase:meta 2024-12-10T00:02:52,563 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2907d75fbb3e=0} racks are {/default-rack=0} 2024-12-10T00:02:52,563 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T00:02:52,563 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T00:02:52,563 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T00:02:52,563 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T00:02:52,563 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T00:02:52,563 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T00:02:52,564 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T00:02:52,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN}] 2024-12-10T00:02:52,565 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN 2024-12-10T00:02:52,566 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN; state=OFFLINE, location=2907d75fbb3e,36597,1733788971397; forceNewPlan=false, retain=false 2024-12-10T00:02:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:52,717 INFO [2907d75fbb3e:40255 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T00:02:52,718 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d9c786f3802c6e255c34a64099197346, regionState=OPENING, regionLocation=2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:52,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN because future has completed 2024-12-10T00:02:52,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9c786f3802c6e255c34a64099197346, server=2907d75fbb3e,36597,1733788971397}] 2024-12-10T00:02:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:52,883 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:52,883 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d9c786f3802c6e255c34a64099197346, NAME => 'testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:52,884 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733788972533 d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,884 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:52,884 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,884 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,885 INFO [StoreOpener-d9c786f3802c6e255c34a64099197346-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,890 INFO [StoreOpener-d9c786f3802c6e255c34a64099197346-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9c786f3802c6e255c34a64099197346 columnFamilyName family 2024-12-10T00:02:52,890 DEBUG [StoreOpener-d9c786f3802c6e255c34a64099197346-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:52,891 INFO [StoreOpener-d9c786f3802c6e255c34a64099197346-1 {}] regionserver.HStore(327): Store=d9c786f3802c6e255c34a64099197346/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:02:52,891 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,892 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,892 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,893 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,893 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,894 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,901 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:02:52,902 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d9c786f3802c6e255c34a64099197346; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71386839, jitterRate=0.06374679505825043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T00:02:52,902 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:52,903 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d9c786f3802c6e255c34a64099197346: Running coprocessor pre-open hook at 1733788972884Writing region info on filesystem at 1733788972884Initializing all the Stores at 1733788972884Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788972885 (+1 ms)Cleaning up temporary data from old regions at 1733788972893 (+8 ms)Running coprocessor post-open hooks at 1733788972902 (+9 ms)Region opened successfully at 1733788972903 (+1 ms) 2024-12-10T00:02:52,904 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346., pid=6, masterSystemTime=1733788972881 2024-12-10T00:02:52,906 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:52,906 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:52,907 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d9c786f3802c6e255c34a64099197346, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:52,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9c786f3802c6e255c34a64099197346, server=2907d75fbb3e,36597,1733788971397 because future has completed 2024-12-10T00:02:52,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T00:02:52,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d9c786f3802c6e255c34a64099197346, server=2907d75fbb3e,36597,1733788971397 in 183 msec 2024-12-10T00:02:52,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T00:02:52,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN in 349 msec 2024-12-10T00:02:52,916 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:02:52,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733788972533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788972916"}]},"ts":"1733788972916"} 2024-12-10T00:02:52,918 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733788972533, state=ENABLED in hbase:meta 2024-12-10T00:02:52,919 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:02:52,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733788972533 in 382 msec 2024-12-10T00:02:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:02:53,170 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733788972533 completed 2024-12-10T00:02:53,170 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733788972533 get assigned. Timeout = 60000ms 2024-12-10T00:02:53,170 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T00:02:53,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733788972533 assigned to meta. Checking AM states. 2024-12-10T00:02:53,175 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T00:02:53,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733788972533 assigned. 2024-12-10T00:02:53,180 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733788972533', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346., hostname=2907d75fbb3e,36597,1733788971397, seqNum=2] 2024-12-10T00:02:53,186 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9c786f3802c6e255c34a64099197346 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T00:02:53,196 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 79eda302c6b64a1d9d2c00380c0f66b6 2024-12-10T00:02:53,210 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/.tmp/family/79eda302c6b64a1d9d2c00380c0f66b6 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T00:02:53,210 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/.tmp/family/79eda302c6b64a1d9d2c00380c0f66b6 2024-12-10T00:02:53,213 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/.tmp/family/79eda302c6b64a1d9d2c00380c0f66b6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6 2024-12-10T00:02:53,215 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6, delay=943 ms 2024-12-10T00:02:53,216 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T00:02:53,217 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d9c786f3802c6e255c34a64099197346 in 31ms, sequenceid=6, compaction requested=false 2024-12-10T00:02:53,217 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733788972533' 2024-12-10T00:02:53,218 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d9c786f3802c6e255c34a64099197346: 2024-12-10T00:02:53,450 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:02:54,160 DEBUG [hfile-prefetch-1733788970559 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 79eda302c6b64a1d9d2c00380c0f66b6_0. 2024-12-10T00:02:54,161 DEBUG [hfile-prefetch-1733788970559 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 79eda302c6b64a1d9d2c00380c0f66b6_117. 2024-12-10T00:02:54,162 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6, with total blocks 2, and data blocks 1 2024-12-10T00:02:54,162 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T00:02:54,162 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T00:02:54,162 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6. Will sleep 100ms and try the verification again. 2024-12-10T00:02:54,218 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;2907d75fbb3e:36597,5,FailOnTimeoutGroup] 2024-12-10T00:02:54,218 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2907d75fbb3e,36597,1733788971397' ***** 2024-12-10T00:02:54,218 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-10T00:02:54,218 INFO [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T00:02:54,241 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T00:02:54,242 INFO [regionserver/2907d75fbb3e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:54,247 INFO [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence 2024-12-10T00:02:54,247 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T00:02:54,247 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T00:02:54,247 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T00:02:54,247 INFO [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T00:02:54,247 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T00:02:54,257 DEBUG [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1487): Thread RS:1;2907d75fbb3e:36597 started persisting bucket cache to file 2024-12-10T00:02:54,258 DEBUG [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:54,258 DEBUG [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:54,263 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6, with total blocks 2, and data blocks 1 2024-12-10T00:02:54,263 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-10T00:02:54,263 DEBUG [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-10T00:02:54,263 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2293): found block for file 79eda302c6b64a1d9d2c00380c0f66b6 in the backing map. Acquiring read lock for offset 0 2024-12-10T00:02:54,263 DEBUG [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733788974257 2024-12-10T00:02:54,263 DEBUG [hfile-prefetch-1733788970559 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/family/79eda302c6b64a1d9d2c00380c0f66b6 has now been fully cached. 2024-12-10T00:02:54,263 DEBUG [RS:1;2907d75fbb3e:36597 {}] bucket.BucketCache(1509): Thread RS:1;2907d75fbb3e:36597 finished persisting bucket cache to file, renaming 2024-12-10T00:02:54,263 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T00:02:54,264 INFO [RS:1;2907d75fbb3e:36597 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T00:02:54,264 DEBUG [hfile-prefetch-1733788970559 {}] hfile.PrefetchExecutor(121): Prefetch completed for 79eda302c6b64a1d9d2c00380c0f66b6 2024-12-10T00:02:54,264 INFO [RS:1;2907d75fbb3e:36597 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T00:02:54,264 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(3091): Received CLOSE for d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:54,264 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(959): stopping server 2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:54,264 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:02:54,265 INFO [RS:1;2907d75fbb3e:36597 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2907d75fbb3e:36597. 2024-12-10T00:02:54,265 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T00:02:54,265 DEBUG [RS:1;2907d75fbb3e:36597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:54,265 DEBUG [RS:1;2907d75fbb3e:36597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:54,265 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d9c786f3802c6e255c34a64099197346, disabling compactions & flushes 2024-12-10T00:02:54,265 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:54,265 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T00:02:54,265 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:54,265 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T00:02:54,265 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. after waiting 0 ms 2024-12-10T00:02:54,265 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T00:02:54,265 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:54,266 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T00:02:54,266 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T00:02:54,266 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d9c786f3802c6e255c34a64099197346=testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346.} 2024-12-10T00:02:54,266 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T00:02:54,266 DEBUG [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d9c786f3802c6e255c34a64099197346 2024-12-10T00:02:54,267 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T00:02:54,267 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T00:02:54,267 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T00:02:54,267 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T00:02:54,267 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T00:02:54,267 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-10T00:02:54,272 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/default/testBlockEvictionOnGracefulStop1733788972533/d9c786f3802c6e255c34a64099197346/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T00:02:54,272 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:54,272 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d9c786f3802c6e255c34a64099197346: Waiting for close lock at 1733788974265Running coprocessor pre-close hooks at 1733788974265Disabling compacts and flushes for region at 1733788974265Disabling writes for close at 1733788974265Writing region close event to WAL at 1733788974267 (+2 ms)Running coprocessor post-close hooks at 1733788974272 (+5 ms)Closed at 1733788974272 2024-12-10T00:02:54,272 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346. 2024-12-10T00:02:54,277 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 70e5cb00f5d54e06a0ad8d3ba4ebf809 2024-12-10T00:02:54,281 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T00:02:54,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T00:02:54,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:54,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:54,283 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T00:02:54,283 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733788974281 2024-12-10T00:02:54,284 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T00:02:54,290 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/70e5cb00f5d54e06a0ad8d3ba4ebf809 is 209, key is testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346./info:regioninfo/1733788972907/Put/seqid=0 2024-12-10T00:02:54,291 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/70e5cb00f5d54e06a0ad8d3ba4ebf809 2024-12-10T00:02:54,298 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3b04a730d25b49978c1ec54bdac6900e 2024-12-10T00:02:54,310 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/ns/3b04a730d25b49978c1ec54bdac6900e is 43, key is default/ns:d/1733788972495/Put/seqid=0 2024-12-10T00:02:54,310 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/ns/3b04a730d25b49978c1ec54bdac6900e 2024-12-10T00:02:54,317 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 114c8338f8624116806cc221d58fdb9a 2024-12-10T00:02:54,328 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/table/114c8338f8624116806cc221d58fdb9a is 80, key is testBlockEvictionOnGracefulStop1733788972533/table:state/1733788972916/Put/seqid=0 2024-12-10T00:02:54,329 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/table/114c8338f8624116806cc221d58fdb9a 2024-12-10T00:02:54,332 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/70e5cb00f5d54e06a0ad8d3ba4ebf809 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/70e5cb00f5d54e06a0ad8d3ba4ebf809 2024-12-10T00:02:54,333 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/70e5cb00f5d54e06a0ad8d3ba4ebf809, delay=971 ms 2024-12-10T00:02:54,334 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/70e5cb00f5d54e06a0ad8d3ba4ebf809, entries=10, sequenceid=11, filesize=7.0 K 2024-12-10T00:02:54,335 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/ns/3b04a730d25b49978c1ec54bdac6900e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/3b04a730d25b49978c1ec54bdac6900e 2024-12-10T00:02:54,336 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/3b04a730d25b49978c1ec54bdac6900e, delay=972 ms 2024-12-10T00:02:54,337 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/3b04a730d25b49978c1ec54bdac6900e, entries=2, sequenceid=11, filesize=5.0 K 2024-12-10T00:02:54,337 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/table/114c8338f8624116806cc221d58fdb9a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/114c8338f8624116806cc221d58fdb9a 2024-12-10T00:02:54,338 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/114c8338f8624116806cc221d58fdb9a, delay=983 ms 2024-12-10T00:02:54,339 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/114c8338f8624116806cc221d58fdb9a, entries=2, sequenceid=11, filesize=5.3 K 2024-12-10T00:02:54,340 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false 2024-12-10T00:02:54,341 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/70e5cb00f5d54e06a0ad8d3ba4ebf809 2024-12-10T00:02:54,341 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/3b04a730d25b49978c1ec54bdac6900e 2024-12-10T00:02:54,342 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/114c8338f8624116806cc221d58fdb9a 2024-12-10T00:02:54,345 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-10T00:02:54,346 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T00:02:54,346 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:54,346 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733788974266Running coprocessor pre-close hooks at 1733788974267 (+1 ms)Disabling compacts and flushes for region at 1733788974267Disabling writes for close at 1733788974267Obtaining lock to block concurrent updates at 1733788974267Preparing flush snapshotting stores in 1588230740 at 1733788974267Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733788974268 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733788974268Flushing 1588230740/info: creating writer at 1733788974268Flushing 1588230740/info: appending metadata at 1733788974290 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733788974290Flushing 1588230740/ns: creating writer at 1733788974292 (+2 ms)Flushing 1588230740/ns: appending metadata at 1733788974309 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733788974309Flushing 1588230740/table: creating writer at 1733788974312 (+3 ms)Flushing 1588230740/table: appending metadata at 1733788974328 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733788974328Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@636434ec: reopening flushed file at 1733788974332 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3384f8f7: reopening flushed file at 1733788974335 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ee988f: reopening flushed file at 1733788974337 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false at 1733788974340 (+3 ms)Writing region close event to WAL at 1733788974342 (+2 ms)Running coprocessor post-close hooks at 1733788974346 (+4 ms)Closed at 1733788974346 2024-12-10T00:02:54,346 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:54,467 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(976): stopping server 2907d75fbb3e,36597,1733788971397; all regions closed. 2024-12-10T00:02:54,472 DEBUG [RS:1;2907d75fbb3e:36597 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:54,472 INFO [RS:1;2907d75fbb3e:36597 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C36597%2C1733788971397.meta:.meta(num 1733788972424) 2024-12-10T00:02:54,477 DEBUG [RS:1;2907d75fbb3e:36597 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:54,477 INFO [RS:1;2907d75fbb3e:36597 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C36597%2C1733788971397:(num 1733788972073) 2024-12-10T00:02:54,477 DEBUG [RS:1;2907d75fbb3e:36597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:54,477 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:54,477 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:02:54,478 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.ChoreService(370): Chore service for: regionserver/2907d75fbb3e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T00:02:54,478 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:02:54,478 INFO [regionserver/2907d75fbb3e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:02:54,478 INFO [RS:1;2907d75fbb3e:36597 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36597 2024-12-10T00:02:54,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:02:54,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2907d75fbb3e,36597,1733788971397 2024-12-10T00:02:54,508 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:02:54,510 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2907d75fbb3e,36597,1733788971397] 2024-12-10T00:02:54,525 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2907d75fbb3e,36597,1733788971397 already deleted, retry=false 2024-12-10T00:02:54,525 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2907d75fbb3e,36597,1733788971397 on 2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:54,530 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2907d75fbb3e,36597,1733788971397, splitWal=true, meta=true 2024-12-10T00:02:54,535 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2907d75fbb3e,36597,1733788971397, splitWal=true, meta=true 2024-12-10T00:02:54,535 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 2907d75fbb3e,36597,1733788971397 (carryingMeta=true) 2907d75fbb3e,36597,1733788971397/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7babb63b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-10T00:02:54,536 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2907d75fbb3e,36597,1733788971397, splitWal=true, meta=true, isMeta: true 2024-12-10T00:02:54,536 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397-splitting 2024-12-10T00:02:54,536 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397-splitting dir is empty, no logs to split. 2024-12-10T00:02:54,536 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2907d75fbb3e,36597,1733788971397 WAL count=0, meta=true 2024-12-10T00:02:54,538 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397-splitting dir is empty, no logs to split. 2024-12-10T00:02:54,538 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2907d75fbb3e,36597,1733788971397 WAL count=0, meta=true 2024-12-10T00:02:54,538 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 2907d75fbb3e,36597,1733788971397 WAL splitting is done? wals=0, meta=true 2024-12-10T00:02:54,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T00:02:54,540 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:02:54,541 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T00:02:54,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:54,617 INFO [RS:1;2907d75fbb3e:36597 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:02:54,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36597-0x1000d15ef460002, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:54,617 INFO [RS:1;2907d75fbb3e:36597 {}] regionserver.HRegionServer(1031): Exiting; stopping=2907d75fbb3e,36597,1733788971397; zookeeper connection closed. 2024-12-10T00:02:54,617 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@180d6b4a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@180d6b4a 2024-12-10T00:02:54,691 DEBUG [2907d75fbb3e:40255 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T00:02:54,692 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:54,693 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,34505,1733788971363, state=OPENING 2024-12-10T00:02:54,738 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2907d75fbb3e:0 server-side Connection retries=45 2024-12-10T00:02:54,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:54,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:54,738 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:02:54,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:02:54,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:02:54,738 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T00:02:54,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:54,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:54,738 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:02:54,738 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:02:54,738 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:54,738 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:54,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,34505,1733788971363}] 2024-12-10T00:02:54,739 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40045 2024-12-10T00:02:54,741 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40045 connecting to ZooKeeper ensemble=127.0.0.1:60691 2024-12-10T00:02:54,741 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:54,741 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:54,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400450x0, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:02:54,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40045-0x1000d15ef460003 connected 2024-12-10T00:02:54,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-10T00:02:54,750 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-10T00:02:54,751 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T00:02:54,752 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.cache 2024-12-10T00:02:54,752 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T00:02:54,752 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T00:02:54,752 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T00:02:54,752 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T00:02:54,752 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T00:02:54,753 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T00:02:54,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T00:02:54,753 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-10T00:02:54,754 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-10T00:02:54,755 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:02:54,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40045 2024-12-10T00:02:54,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40045 2024-12-10T00:02:54,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40045 2024-12-10T00:02:54,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40045 2024-12-10T00:02:54,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40045 2024-12-10T00:02:54,757 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-10T00:02:54,758 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T00:02:54,758 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-10T00:02:54,758 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence is complete. 2024-12-10T00:02:54,758 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(746): ClusterId : c8cdf554-9edf-496d-87f6-2944d756a9d6 2024-12-10T00:02:54,758 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T00:02:54,768 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T00:02:54,768 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T00:02:54,776 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T00:02:54,776 DEBUG [RS:2;2907d75fbb3e:40045 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200ed0e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-10T00:02:54,788 DEBUG [RS:2;2907d75fbb3e:40045 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2907d75fbb3e:40045 2024-12-10T00:02:54,788 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T00:02:54,788 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T00:02:54,788 DEBUG [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T00:02:54,789 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,40255,1733788971194 with port=40045, startcode=1733788974737 2024-12-10T00:02:54,789 DEBUG [RS:2;2907d75fbb3e:40045 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T00:02:54,791 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59335, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T00:02:54,791 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40255 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:54,792 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40255 {}] master.ServerManager(517): Registering regionserver=2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:54,793 DEBUG [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c 2024-12-10T00:02:54,793 DEBUG [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T00:02:54,793 DEBUG [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T00:02:54,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:02:54,801 DEBUG [RS:2;2907d75fbb3e:40045 {}] zookeeper.ZKUtil(111): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:54,801 WARN [RS:2;2907d75fbb3e:40045 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:02:54,801 INFO [RS:2;2907d75fbb3e:40045 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:54,801 DEBUG [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:54,801 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2907d75fbb3e,40045,1733788974737] 2024-12-10T00:02:54,806 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T00:02:54,809 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T00:02:54,810 INFO [RS:2;2907d75fbb3e:40045 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T00:02:54,810 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,810 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T00:02:54,811 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T00:02:54,811 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,811 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,812 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,812 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,812 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,812 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:02:54,812 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:54,812 DEBUG [RS:2;2907d75fbb3e:40045 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:02:54,812 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,812 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,812 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,813 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,813 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,813 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40045,1733788974737-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:02:54,827 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T00:02:54,827 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,40045,1733788974737-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,828 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,828 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.Replication(171): 2907d75fbb3e,40045,1733788974737 started 2024-12-10T00:02:54,839 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:02:54,839 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(1482): Serving as 2907d75fbb3e,40045,1733788974737, RpcServer on 2907d75fbb3e/172.17.0.2:40045, sessionid=0x1000d15ef460003 2024-12-10T00:02:54,840 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T00:02:54,840 DEBUG [RS:2;2907d75fbb3e:40045 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:54,840 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,40045,1733788974737' 2024-12-10T00:02:54,840 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T00:02:54,840 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T00:02:54,841 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T00:02:54,841 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T00:02:54,841 DEBUG [RS:2;2907d75fbb3e:40045 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:54,841 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,40045,1733788974737' 2024-12-10T00:02:54,841 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T00:02:54,841 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T00:02:54,842 DEBUG [RS:2;2907d75fbb3e:40045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T00:02:54,842 INFO [RS:2;2907d75fbb3e:40045 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T00:02:54,842 INFO [RS:2;2907d75fbb3e:40045 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T00:02:54,892 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T00:02:54,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54799, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T00:02:54,899 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T00:02:54,899 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:02:54,901 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T00:02:54,902 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C34505%2C1733788971363.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,34505,1733788971363, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:54,918 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,34505,1733788971363/2907d75fbb3e%2C34505%2C1733788971363.meta.1733788974903.meta 2024-12-10T00:02:54,918 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:54,918 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:02:54,919 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T00:02:54,919 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T00:02:54,919 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T00:02:54,919 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T00:02:54,919 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:02:54,919 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T00:02:54,919 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T00:02:54,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T00:02:54,920 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T00:02:54,920 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:54,922 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/70e5cb00f5d54e06a0ad8d3ba4ebf809, delay=912 ms 2024-12-10T00:02:54,922 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/70e5cb00f5d54e06a0ad8d3ba4ebf809 2024-12-10T00:02:54,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:54,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T00:02:54,923 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T00:02:54,923 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:54,925 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/3b04a730d25b49978c1ec54bdac6900e, delay=912 ms 2024-12-10T00:02:54,926 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/3b04a730d25b49978c1ec54bdac6900e 2024-12-10T00:02:54,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:54,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T00:02:54,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T00:02:54,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:54,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:54,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T00:02:54,927 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T00:02:54,927 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:02:54,928 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/114c8338f8624116806cc221d58fdb9a, delay=973 ms 2024-12-10T00:02:54,929 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/114c8338f8624116806cc221d58fdb9a 2024-12-10T00:02:54,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:02:54,929 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T00:02:54,930 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:54,930 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740 2024-12-10T00:02:54,930 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T00:02:54,931 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T00:02:54,931 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:02:54,932 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T00:02:54,932 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59760437, jitterRate=-0.10950009524822235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:02:54,932 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T00:02:54,933 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733788974919Writing region info on filesystem at 1733788974919Initializing all the Stores at 1733788974919Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788974919Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788974920 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788974920Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788974920Cleaning up temporary data from old regions at 1733788974931 (+11 ms)Running coprocessor post-open hooks at 1733788974932 (+1 ms)Region opened successfully at 1733788974933 (+1 ms) 2024-12-10T00:02:54,933 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733788974891 2024-12-10T00:02:54,936 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T00:02:54,936 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T00:02:54,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:54,937 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,34505,1733788971363, state=OPEN 2024-12-10T00:02:54,943 INFO [RS:2;2907d75fbb3e:40045 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:02:54,943 INFO [RS:2;2907d75fbb3e:40045 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2907d75fbb3e%2C40045%2C1733788974737, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,40045,1733788974737, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs, maxLogs=55 2024-12-10T00:02:54,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:54,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:54,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:02:54,950 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:54,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:54,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:54,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:02:54,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-10T00:02:54,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,34505,1733788971363 in 211 msec 2024-12-10T00:02:54,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T00:02:54,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 414 msec 2024-12-10T00:02:54,955 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 2907d75fbb3e,36597,1733788971397 had 2 regions 2024-12-10T00:02:54,956 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2907d75fbb3e,36597,1733788971397, splitWal=true, meta=true, isMeta: false 2024-12-10T00:02:54,956 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397-splitting dir is empty, no logs to split. 2024-12-10T00:02:54,957 INFO [PEWorker-3 {}] master.SplitWALManager(105): 2907d75fbb3e,36597,1733788971397 WAL count=0, meta=false 2024-12-10T00:02:54,958 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,36597,1733788971397-splitting dir is empty, no logs to split. 2024-12-10T00:02:54,958 INFO [PEWorker-3 {}] master.SplitWALManager(105): 2907d75fbb3e,36597,1733788971397 WAL count=0, meta=false 2024-12-10T00:02:54,958 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 2907d75fbb3e,36597,1733788971397 WAL splitting is done? wals=0, meta=false 2024-12-10T00:02:54,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN}] 2024-12-10T00:02:54,961 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN 2024-12-10T00:02:54,962 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T00:02:54,965 INFO [RS:2;2907d75fbb3e:40045 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/WALs/2907d75fbb3e,40045,1733788974737/2907d75fbb3e%2C40045%2C1733788974737.1733788974944 2024-12-10T00:02:54,965 DEBUG [RS:2;2907d75fbb3e:40045 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T00:02:55,113 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(204): Hosts are {2907d75fbb3e=0} racks are {/default-rack=0} 2024-12-10T00:02:55,113 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T00:02:55,113 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T00:02:55,113 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T00:02:55,113 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T00:02:55,113 INFO [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T00:02:55,114 INFO [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T00:02:55,114 DEBUG [2907d75fbb3e:40255 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T00:02:55,114 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d9c786f3802c6e255c34a64099197346, regionState=OPENING, regionLocation=2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:55,116 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2907d75fbb3e:36597 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2907d75fbb3e/172.17.0.2:36597 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:02:55,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,36597,1733788971397, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,36597,1733788971397, seqNum=-1, error=java.net.ConnectException: Call to address=2907d75fbb3e:36597 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2907d75fbb3e/172.17.0.2:36597 2024-12-10T00:02:55,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,36597,1733788971397, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-10T00:02:55,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,36597,1733788971397, seqNum=-1 from cache 2024-12-10T00:02:55,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 2907d75fbb3e:36597 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2907d75fbb3e/172.17.0.2:36597 2024-12-10T00:02:55,228 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:02:55,228 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,34505,1733788971363, seqNum=-1] 2024-12-10T00:02:55,229 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:02:55,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:02:55,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346, ASSIGN because future has completed 2024-12-10T00:02:55,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9c786f3802c6e255c34a64099197346, server=2907d75fbb3e,34505,1733788971363}] 2024-12-10T00:02:55,282 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T00:02:55,282 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T00:02:55,285 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T00:02:55,285 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T00:02:55,285 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T00:02:55,285 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T00:02:55,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T00:02:55,340 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:02:55,340 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:55,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:55,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:55,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:02:55,341 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T00:02:55,341 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1433448022, stopped=false 2024-12-10T00:02:55,341 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2907d75fbb3e,40255,1733788971194 2024-12-10T00:02:55,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:55,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:55,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:02:55,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:55,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:55,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:55,384 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T00:02:55,384 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:02:55,385 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:55,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:55,385 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:55,386 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:55,386 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2907d75fbb3e,34505,1733788971363' ***** 2024-12-10T00:02:55,386 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T00:02:55,386 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2907d75fbb3e,40045,1733788974737' ***** 2024-12-10T00:02:55,386 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T00:02:55,386 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:02:55,386 INFO [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T00:02:55,387 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T00:02:55,387 INFO [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence 2024-12-10T00:02:55,387 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T00:02:55,388 INFO [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T00:02:55,389 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T00:02:55,389 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T00:02:55,389 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T00:02:55,389 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T00:02:55,389 INFO [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4396f5a-ccf4-02b8-b2b8-af4fbdfd9703/bucket.persistence 2024-12-10T00:02:55,389 INFO [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T00:02:55,389 INFO [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T00:02:55,389 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T00:02:55,389 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T00:02:55,388 DEBUG [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(271): Request to 2907d75fbb3e,34505,1733788971363 failed, try=0 org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2907d75fbb3e:34505 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-12-10T00:02:55,390 WARN [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): 2907d75fbb3e,34505,1733788971363 tells us DoNotRetry due to org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2907d75fbb3e:34505 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException, try=0, give up 2024-12-10T00:02:55,390 WARN [RSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(139): The remote operation pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9c786f3802c6e255c34a64099197346, server=2907d75fbb3e,34505,1733788971363 for region state=OPENING, location=2907d75fbb3e,34505,1733788971363, table=testBlockEvictionOnGracefulStop1733788972533, region=d9c786f3802c6e255c34a64099197346 to server 2907d75fbb3e,34505,1733788971363 failed org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2907d75fbb3e:34505 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-12-10T00:02:55,399 DEBUG [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1487): Thread RS:0;2907d75fbb3e:34505 started persisting bucket cache to file 2024-12-10T00:02:55,399 DEBUG [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1487): Thread RS:2;2907d75fbb3e:40045 started persisting bucket cache to file 2024-12-10T00:02:55,399 DEBUG [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:55,399 DEBUG [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T00:02:55,399 DEBUG [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:55,399 DEBUG [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T00:02:55,402 DEBUG [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T00:02:55,402 DEBUG [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T00:02:55,402 DEBUG [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733788975399 2024-12-10T00:02:55,402 DEBUG [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733788975399 2024-12-10T00:02:55,402 DEBUG [RS:0;2907d75fbb3e:34505 {}] bucket.BucketCache(1509): Thread RS:0;2907d75fbb3e:34505 finished persisting bucket cache to file, renaming 2024-12-10T00:02:55,402 DEBUG [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1509): Thread RS:2;2907d75fbb3e:40045 finished persisting bucket cache to file, renaming 2024-12-10T00:02:55,402 WARN [RS:2;2907d75fbb3e:40045 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T00:02:55,402 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T00:02:55,402 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T00:02:55,402 INFO [RS:0;2907d75fbb3e:34505 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T00:02:55,402 INFO [RS:2;2907d75fbb3e:40045 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T00:02:55,402 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T00:02:55,402 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T00:02:55,402 INFO [RS:0;2907d75fbb3e:34505 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T00:02:55,402 INFO [RS:2;2907d75fbb3e:40045 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T00:02:55,402 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(959): stopping server 2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:55,402 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(959): stopping server 2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:55,402 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:02:55,402 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:02:55,403 INFO [RS:2;2907d75fbb3e:40045 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2907d75fbb3e:40045. 2024-12-10T00:02:55,403 INFO [RS:0;2907d75fbb3e:34505 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2907d75fbb3e:34505. 2024-12-10T00:02:55,403 DEBUG [RS:2;2907d75fbb3e:40045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:55,403 DEBUG [RS:0;2907d75fbb3e:34505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:02:55,403 DEBUG [RS:2;2907d75fbb3e:40045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:55,403 DEBUG [RS:0;2907d75fbb3e:34505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:55,403 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T00:02:55,403 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T00:02:55,403 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T00:02:55,403 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T00:02:55,403 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(976): stopping server 2907d75fbb3e,40045,1733788974737; all regions closed. 2024-12-10T00:02:55,403 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T00:02:55,403 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T00:02:55,403 DEBUG [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:02:55,403 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T00:02:55,403 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T00:02:55,404 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T00:02:55,404 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T00:02:55,404 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T00:02:55,404 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T00:02:55,404 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-12-10T00:02:55,405 DEBUG [RS:2;2907d75fbb3e:40045 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C40045%2C1733788974737:(num 1733788974944) 2024-12-10T00:02:55,405 DEBUG [RS:2;2907d75fbb3e:40045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.ChoreService(370): Chore service for: regionserver/2907d75fbb3e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T00:02:55,405 INFO [regionserver/2907d75fbb3e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:02:55,405 INFO [RS:2;2907d75fbb3e:40045 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40045 2024-12-10T00:02:55,413 INFO [regionserver/2907d75fbb3e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:55,417 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 086cd919887749cbbf9b3bf56a2071b8 2024-12-10T00:02:55,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:02:55,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2907d75fbb3e,40045,1733788974737 2024-12-10T00:02:55,426 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:02:55,426 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2907d75fbb3e,40045,1733788974737] 2024-12-10T00:02:55,430 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/086cd919887749cbbf9b3bf56a2071b8 is 209, key is testBlockEvictionOnGracefulStop1733788972533,,1733788972537.d9c786f3802c6e255c34a64099197346./info:regioninfo/1733788975114/Put/seqid=0 2024-12-10T00:02:55,431 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/086cd919887749cbbf9b3bf56a2071b8 2024-12-10T00:02:55,433 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/.tmp/info/086cd919887749cbbf9b3bf56a2071b8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/086cd919887749cbbf9b3bf56a2071b8 2024-12-10T00:02:55,434 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/086cd919887749cbbf9b3bf56a2071b8, delay=984 ms 2024-12-10T00:02:55,435 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/086cd919887749cbbf9b3bf56a2071b8, entries=3, sequenceid=18, filesize=5.9 K 2024-12-10T00:02:55,436 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=18, compaction requested=false 2024-12-10T00:02:55,436 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/70e5cb00f5d54e06a0ad8d3ba4ebf809 2024-12-10T00:02:55,437 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/info/086cd919887749cbbf9b3bf56a2071b8 2024-12-10T00:02:55,437 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/ns/3b04a730d25b49978c1ec54bdac6900e 2024-12-10T00:02:55,438 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/table/114c8338f8624116806cc221d58fdb9a 2024-12-10T00:02:55,441 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-12-10T00:02:55,441 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2907d75fbb3e,40045,1733788974737 already deleted, retry=false 2024-12-10T00:02:55,442 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T00:02:55,442 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2907d75fbb3e,40045,1733788974737 expired; onlineServers=1 2024-12-10T00:02:55,442 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:55,442 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733788975403Running coprocessor pre-close hooks at 1733788975403Disabling compacts and flushes for region at 1733788975403Disabling writes for close at 1733788975404 (+1 ms)Obtaining lock to block concurrent updates at 1733788975404Preparing flush snapshotting stores in 1588230740 at 1733788975404Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1733788975404Flushing stores of hbase:meta,,1.1588230740 at 1733788975405 (+1 ms)Flushing 1588230740/info: creating writer at 1733788975405Flushing 1588230740/info: appending metadata at 1733788975429 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733788975430 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31fb85a1: reopening flushed file at 1733788975433 (+3 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=18, compaction requested=false at 1733788975436 (+3 ms)Writing region close event to WAL at 1733788975438 (+2 ms)Running coprocessor post-close hooks at 1733788975441 (+3 ms)Closed at 1733788975442 (+1 ms) 2024-12-10T00:02:55,442 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T00:02:55,446 INFO [regionserver/2907d75fbb3e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:55,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:55,534 INFO [RS:2;2907d75fbb3e:40045 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:02:55,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40045-0x1000d15ef460003, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:55,534 INFO [RS:2;2907d75fbb3e:40045 {}] regionserver.HRegionServer(1031): Exiting; stopping=2907d75fbb3e,40045,1733788974737; zookeeper connection closed. 2024-12-10T00:02:55,534 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2018cc42 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2018cc42 2024-12-10T00:02:55,604 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(976): stopping server 2907d75fbb3e,34505,1733788971363; all regions closed. 2024-12-10T00:02:55,605 DEBUG [RS:0;2907d75fbb3e:34505 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:55,606 INFO [RS:0;2907d75fbb3e:34505 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C34505%2C1733788971363.meta:.meta(num 1733788974903) 2024-12-10T00:02:55,607 DEBUG [RS:0;2907d75fbb3e:34505 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/oldWALs 2024-12-10T00:02:55,607 INFO [RS:0;2907d75fbb3e:34505 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C34505%2C1733788971363:(num 1733788972087) 2024-12-10T00:02:55,607 DEBUG [RS:0;2907d75fbb3e:34505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:02:55,607 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:02:55,607 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:02:55,607 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.ChoreService(370): Chore service for: regionserver/2907d75fbb3e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T00:02:55,608 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:02:55,608 INFO [regionserver/2907d75fbb3e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:02:55,608 INFO [RS:0;2907d75fbb3e:34505 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34505 2024-12-10T00:02:55,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2907d75fbb3e,34505,1733788971363 2024-12-10T00:02:55,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:02:55,617 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:02:55,625 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2907d75fbb3e,34505,1733788971363] 2024-12-10T00:02:55,633 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2907d75fbb3e,34505,1733788971363 already deleted, retry=false 2024-12-10T00:02:55,633 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2907d75fbb3e,34505,1733788971363 expired; onlineServers=0 2024-12-10T00:02:55,633 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2907d75fbb3e,40255,1733788971194' ***** 2024-12-10T00:02:55,633 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T00:02:55,633 INFO [M:0;2907d75fbb3e:40255 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:02:55,633 INFO [M:0;2907d75fbb3e:40255 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:02:55,634 DEBUG [M:0;2907d75fbb3e:40255 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T00:02:55,634 DEBUG [M:0;2907d75fbb3e:40255 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T00:02:55,634 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T00:02:55,634 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788971771 {}] cleaner.HFileCleaner(306): Exit Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788971771,5,FailOnTimeoutGroup] 2024-12-10T00:02:55,634 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788971771 {}] cleaner.HFileCleaner(306): Exit Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788971771,5,FailOnTimeoutGroup] 2024-12-10T00:02:55,634 INFO [M:0;2907d75fbb3e:40255 {}] hbase.ChoreService(370): Chore service for: master/2907d75fbb3e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T00:02:55,634 INFO [M:0;2907d75fbb3e:40255 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:02:55,634 DEBUG [M:0;2907d75fbb3e:40255 {}] master.HMaster(1795): Stopping service threads 2024-12-10T00:02:55,634 INFO [M:0;2907d75fbb3e:40255 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T00:02:55,634 INFO [M:0;2907d75fbb3e:40255 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T00:02:55,635 INFO [M:0;2907d75fbb3e:40255 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T00:02:55,635 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T00:02:55,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T00:02:55,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:02:55,642 DEBUG [M:0;2907d75fbb3e:40255 {}] zookeeper.ZKUtil(347): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T00:02:55,642 WARN [M:0;2907d75fbb3e:40255 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T00:02:55,642 INFO [M:0;2907d75fbb3e:40255 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/.lastflushedseqids 2024-12-10T00:02:55,647 INFO [M:0;2907d75fbb3e:40255 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T00:02:55,648 INFO [M:0;2907d75fbb3e:40255 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T00:02:55,648 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T00:02:55,648 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:55,648 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:55,648 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T00:02:55,648 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:55,648 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.95 KB heapSize=44.17 KB 2024-12-10T00:02:55,657 DEBUG [M:0;2907d75fbb3e:40255 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dffeb790c8b146c8bd77d7078fa1aa53 2024-12-10T00:02:55,668 DEBUG [M:0;2907d75fbb3e:40255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dffeb790c8b146c8bd77d7078fa1aa53 is 82, key is hbase:meta,,1/info:regioninfo/1733788974936/Put/seqid=0 2024-12-10T00:02:55,669 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dffeb790c8b146c8bd77d7078fa1aa53 2024-12-10T00:02:55,676 DEBUG [M:0;2907d75fbb3e:40255 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0811f39db81245089e05dad85a5159c4 2024-12-10T00:02:55,690 DEBUG [M:0;2907d75fbb3e:40255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0811f39db81245089e05dad85a5159c4 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733788972920/Put/seqid=0 2024-12-10T00:02:55,690 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.73 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0811f39db81245089e05dad85a5159c4 2024-12-10T00:02:55,698 DEBUG [M:0;2907d75fbb3e:40255 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 71a5811510034aa088c3d6f5d43b8e4d 2024-12-10T00:02:55,711 DEBUG [M:0;2907d75fbb3e:40255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71a5811510034aa088c3d6f5d43b8e4d is 69, key is 2907d75fbb3e,34505,1733788971363/rs:state/1733788971880/Put/seqid=0 2024-12-10T00:02:55,712 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71a5811510034aa088c3d6f5d43b8e4d 2024-12-10T00:02:55,714 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 71a5811510034aa088c3d6f5d43b8e4d 2024-12-10T00:02:55,714 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dffeb790c8b146c8bd77d7078fa1aa53 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dffeb790c8b146c8bd77d7078fa1aa53 2024-12-10T00:02:55,716 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dffeb790c8b146c8bd77d7078fa1aa53, entries=14, sequenceid=90, filesize=5.9 K 2024-12-10T00:02:55,717 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0811f39db81245089e05dad85a5159c4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0811f39db81245089e05dad85a5159c4 2024-12-10T00:02:55,718 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0811f39db81245089e05dad85a5159c4, entries=11, sequenceid=90, filesize=7.8 K 2024-12-10T00:02:55,719 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71a5811510034aa088c3d6f5d43b8e4d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/71a5811510034aa088c3d6f5d43b8e4d 2024-12-10T00:02:55,720 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 71a5811510034aa088c3d6f5d43b8e4d 2024-12-10T00:02:55,720 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/236da225-b062-8bb6-061d-ef6abd788bfa/866a3501-aa51-4444-1f08-33f64649a05c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/71a5811510034aa088c3d6f5d43b8e4d, entries=3, sequenceid=90, filesize=5.3 K 2024-12-10T00:02:55,721 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=90, compaction requested=false 2024-12-10T00:02:55,723 INFO [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:02:55,723 DEBUG [M:0;2907d75fbb3e:40255 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733788975648Disabling compacts and flushes for region at 1733788975648Disabling writes for close at 1733788975648Obtaining lock to block concurrent updates at 1733788975648Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733788975648Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35792, getHeapSize=45168, getOffHeapSize=0, getCellsCount=111 at 1733788975649 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733788975649Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733788975649Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733788975667 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733788975667Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733788975671 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733788975689 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733788975689Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733788975692 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733788975710 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733788975710Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@635ce565: reopening flushed file at 1733788975714 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cefdb: reopening flushed file at 1733788975716 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15134aaf: reopening flushed file at 1733788975718 (+2 ms)Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=90, compaction requested=false at 1733788975721 (+3 ms)Writing region close event to WAL at 1733788975723 (+2 ms)Closed at 1733788975723 2024-12-10T00:02:55,724 INFO [M:0;2907d75fbb3e:40255 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T00:02:55,724 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:02:55,724 INFO [M:0;2907d75fbb3e:40255 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40255 2024-12-10T00:02:55,724 INFO [M:0;2907d75fbb3e:40255 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:02:55,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:55,725 INFO [RS:0;2907d75fbb3e:34505 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:02:55,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34505-0x1000d15ef460001, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:55,725 INFO [RS:0;2907d75fbb3e:34505 {}] regionserver.HRegionServer(1031): Exiting; stopping=2907d75fbb3e,34505,1733788971363; zookeeper connection closed. 2024-12-10T00:02:55,725 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5f6f8de4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5f6f8de4 2024-12-10T00:02:55,725 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-10T00:02:55,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:55,842 INFO [M:0;2907d75fbb3e:40255 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:02:55,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40255-0x1000d15ef460000, quorum=127.0.0.1:60691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:02:55,849 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T00:02:55,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T00:02:55,852 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:02:55,859 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=79 (was 69) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4605 (was 4706)