2024-12-07 06:44:59,215 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 06:44:59,243 main DEBUG Took 0.025005 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 06:44:59,243 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 06:44:59,244 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 06:44:59,245 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 06:44:59,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,254 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 06:44:59,268 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,270 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,271 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,271 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,272 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,273 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,274 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,274 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,276 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,277 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,277 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,278 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,279 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,280 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,281 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:44:59,283 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,283 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 06:44:59,285 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:44:59,286 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 06:44:59,288 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 06:44:59,289 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 06:44:59,290 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 06:44:59,291 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 06:44:59,301 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 06:44:59,304 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 06:44:59,307 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 06:44:59,307 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 06:44:59,308 main DEBUG createAppenders(={Console}) 2024-12-07 06:44:59,308 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-07 06:44:59,309 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 06:44:59,309 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-07 06:44:59,310 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 06:44:59,310 main DEBUG OutputStream closed 2024-12-07 06:44:59,310 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 06:44:59,311 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 06:44:59,311 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-07 06:44:59,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 06:44:59,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 06:44:59,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 06:44:59,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 06:44:59,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 06:44:59,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 06:44:59,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 06:44:59,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 06:44:59,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 06:44:59,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 06:44:59,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 06:44:59,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 06:44:59,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 06:44:59,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 06:44:59,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 06:44:59,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 06:44:59,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 06:44:59,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 06:44:59,415 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 06:44:59,415 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-07 06:44:59,415 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 06:44:59,416 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-07T06:44:59,432 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-07 06:44:59,435 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 06:44:59,435 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T06:44:59,681 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190 2024-12-07T06:44:59,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=6181 2024-12-07T06:44:59,797 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T06:44:59,887 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:44:59,901 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/cluster_e2649f12-6e6a-e239-af8a-584c870802e1, deleteOnExit=true 2024-12-07T06:44:59,999 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/cluster_e2649f12-6e6a-e239-af8a-584c870802e1/zookeeper_0, clientPort=60383, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/cluster_e2649f12-6e6a-e239-af8a-584c870802e1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/cluster_e2649f12-6e6a-e239-af8a-584c870802e1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T06:45:00,013 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60383 2024-12-07T06:45:00,013 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:00,097 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c with version=8 2024-12-07T06:45:00,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/hbase-staging 2024-12-07T06:45:00,195 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T06:45:00,462 INFO [Time-limited test {}] client.ConnectionUtils(128): master/61c02eafbb40:0 server-side Connection retries=45 2024-12-07T06:45:00,473 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:00,474 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:00,481 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:45:00,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:00,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:45:00,636 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T06:45:00,725 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T06:45:00,761 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T06:45:00,772 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T06:45:00,776 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:45:00,817 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 16547 (auto-detected) 2024-12-07T06:45:00,819 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-07T06:45:00,843 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35719 2024-12-07T06:45:00,866 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35719 connecting to ZooKeeper ensemble=127.0.0.1:60383 2024-12-07T06:45:00,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357190x0, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:45:00,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35719-0x1018bab34550000 connected 2024-12-07T06:45:00,933 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:00,934 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:00,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:00,947 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c, hbase.cluster.distributed=false 2024-12-07T06:45:00,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:45:00,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35719 2024-12-07T06:45:00,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35719 2024-12-07T06:45:00,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35719 2024-12-07T06:45:00,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35719 2024-12-07T06:45:00,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35719 2024-12-07T06:45:01,067 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c02eafbb40:0 server-side Connection retries=45 2024-12-07T06:45:01,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:01,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:01,070 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:45:01,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:01,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:45:01,073 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:45:01,075 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:45:01,076 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43483 2024-12-07T06:45:01,078 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43483 connecting to ZooKeeper ensemble=127.0.0.1:60383 2024-12-07T06:45:01,079 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:01,080 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:01,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434830x0, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:45:01,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43483-0x1018bab34550001 connected 2024-12-07T06:45:01,085 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:01,091 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:45:01,103 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.cache 2024-12-07T06:45:01,108 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:45:01,109 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:45:01,111 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:45:01,116 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:45:01,116 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:45:01,121 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:45:01,126 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:45:01,127 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:45:01,127 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.persistence is complete. 2024-12-07T06:45:01,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:45:01,134 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:45:01,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43483 2024-12-07T06:45:01,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43483 2024-12-07T06:45:01,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43483 2024-12-07T06:45:01,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43483 2024-12-07T06:45:01,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43483 2024-12-07T06:45:01,162 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c02eafbb40:0 server-side Connection retries=45 2024-12-07T06:45:01,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:01,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:01,163 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:45:01,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:01,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:45:01,163 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:45:01,164 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:45:01,165 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33241 2024-12-07T06:45:01,166 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33241 connecting to ZooKeeper ensemble=127.0.0.1:60383 2024-12-07T06:45:01,167 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:01,168 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:01,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:332410x0, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:45:01,177 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:332410x0, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:01,177 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33241-0x1018bab34550002 connected 2024-12-07T06:45:01,178 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:45:01,179 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.cache 2024-12-07T06:45:01,179 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:45:01,179 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:45:01,179 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:45:01,180 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:45:01,180 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:45:01,180 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:45:01,181 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:45:01,181 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.persistence is complete. 2024-12-07T06:45:01,181 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:45:01,182 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:45:01,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:45:01,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33241 2024-12-07T06:45:01,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33241 2024-12-07T06:45:01,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33241 2024-12-07T06:45:01,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33241 2024-12-07T06:45:01,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33241 2024-12-07T06:45:01,213 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c02eafbb40:35719 2024-12-07T06:45:01,214 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c02eafbb40,35719,1733553900265 2024-12-07T06:45:01,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:01,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:01,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:01,227 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c02eafbb40,35719,1733553900265 2024-12-07T06:45:01,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:45:01,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:45:01,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:01,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:01,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:01,254 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T06:45:01,256 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c02eafbb40,35719,1733553900265 from backup master directory 2024-12-07T06:45:01,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:01,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:01,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c02eafbb40,35719,1733553900265 2024-12-07T06:45:01,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:01,261 WARN [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:45:01,261 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c02eafbb40,35719,1733553900265 2024-12-07T06:45:01,263 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T06:45:01,265 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T06:45:01,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/hbase.id] with ID: efb50fce-db46-4c2a-86da-374c5a8fffde 2024-12-07T06:45:01,275 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.tmp/hbase.id 2024-12-07T06:45:01,281 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/hbase.id] 2024-12-07T06:45:01,320 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:01,325 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T06:45:01,354 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-12-07T06:45:01,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:01,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:01,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:01,386 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:45:01,389 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T06:45:01,436 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T06:45:01,442 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:01,505 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store 2024-12-07T06:45:01,533 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T06:45:01,538 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:01,540 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:45:01,540 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:01,541 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:01,543 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:45:01,543 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:01,543 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:01,544 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733553901540Disabling compacts and flushes for region at 1733553901540Disabling writes for close at 1733553901543 (+3 ms)Writing region close event to WAL at 1733553901543Closed at 1733553901543 2024-12-07T06:45:01,545 WARN [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/.initializing 2024-12-07T06:45:01,546 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/WALs/61c02eafbb40,35719,1733553900265 2024-12-07T06:45:01,563 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:45:01,576 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c02eafbb40%2C35719%2C1733553900265, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/WALs/61c02eafbb40,35719,1733553900265, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/oldWALs, maxLogs=10 2024-12-07T06:45:01,613 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/WALs/61c02eafbb40,35719,1733553900265/61c02eafbb40%2C35719%2C1733553900265.1733553901580 2024-12-07T06:45:01,614 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:01,615 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:01,615 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:01,619 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,620 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T06:45:01,703 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:01,705 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:01,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T06:45:01,712 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:01,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:01,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T06:45:01,718 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:01,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:01,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T06:45:01,726 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:01,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:01,727 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,732 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,732 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,736 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,737 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,741 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:45:01,743 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:01,755 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:45:01,756 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73786744, jitterRate=0.09950816631317139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:45:01,766 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733553901632Initializing all the Stores at 1733553901633 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553901633Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553901635 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553901635Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553901635Cleaning up temporary data from old regions at 1733553901737 (+102 ms)Region opened successfully at 1733553901765 (+28 ms) 2024-12-07T06:45:01,770 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T06:45:01,805 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b4acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c02eafbb40/172.17.0.2:0 2024-12-07T06:45:01,846 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T06:45:01,858 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T06:45:01,859 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T06:45:01,863 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T06:45:01,864 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T06:45:01,872 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-07T06:45:01,872 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T06:45:01,920 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T06:45:01,941 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T06:45:01,947 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T06:45:01,952 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T06:45:01,973 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T06:45:01,977 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T06:45:01,984 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T06:45:02,010 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T06:45:02,014 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T06:45:02,021 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T06:45:02,025 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T06:45:02,131 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T06:45:02,137 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T06:45:02,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:02,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:02,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:02,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,226 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c02eafbb40,35719,1733553900265, sessionid=0x1018bab34550000, setting cluster-up flag (Was=false) 2024-12-07T06:45:02,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,332 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T06:45:02,337 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c02eafbb40,35719,1733553900265 2024-12-07T06:45:02,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:02,370 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T06:45:02,381 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c02eafbb40,35719,1733553900265 2024-12-07T06:45:02,418 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T06:45:02,519 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(746): ClusterId : efb50fce-db46-4c2a-86da-374c5a8fffde 2024-12-07T06:45:02,548 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:45:02,548 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(746): ClusterId : efb50fce-db46-4c2a-86da-374c5a8fffde 2024-12-07T06:45:02,549 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:45:02,566 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:45:02,566 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:45:02,581 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:45:02,582 DEBUG [RS:1;61c02eafbb40:33241 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c69bac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c02eafbb40/172.17.0.2:0 2024-12-07T06:45:02,599 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:45:02,599 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:45:02,611 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:45:02,612 DEBUG [RS:0;61c02eafbb40:43483 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fe5f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c02eafbb40/172.17.0.2:0 2024-12-07T06:45:02,706 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;61c02eafbb40:33241 2024-12-07T06:45:02,724 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:45:02,724 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:45:02,724 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:45:02,741 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=33241, startcode=1733553901162 2024-12-07T06:45:02,800 DEBUG [RS:1;61c02eafbb40:33241 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:45:02,815 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c02eafbb40:43483 2024-12-07T06:45:02,881 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:45:02,882 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:45:02,882 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:45:02,884 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=43483, startcode=1733553901028 2024-12-07T06:45:02,887 DEBUG [RS:0;61c02eafbb40:43483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:45:03,185 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:03,304 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48029, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:45:03,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50023, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:45:03,313 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-07T06:45:03,398 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-07T06:45:03,419 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T06:45:03,432 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T06:45:03,432 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T06:45:03,432 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T06:45:03,432 WARN [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-07T06:45:03,433 WARN [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-07T06:45:03,504 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c02eafbb40,35719,1733553900265 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T06:45:03,522 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:03,523 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:03,523 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:03,523 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:03,524 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c02eafbb40:0, corePoolSize=10, maxPoolSize=10 2024-12-07T06:45:03,524 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:03,524 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c02eafbb40:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:45:03,524 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:03,535 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=43483, startcode=1733553901028 2024-12-07T06:45:03,542 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-07T06:45:03,544 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T06:45:03,544 WARN [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 200 ms and then retrying. 2024-12-07T06:45:03,603 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=33241, startcode=1733553901162 2024-12-07T06:45:03,605 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-07T06:45:03,607 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T06:45:03,607 WARN [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 200 ms and then retrying. 2024-12-07T06:45:03,616 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:03,616 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T06:45:03,623 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:03,623 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T06:45:03,696 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733553933696 2024-12-07T06:45:03,698 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T06:45:03,700 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T06:45:03,703 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T06:45:03,704 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T06:45:03,704 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T06:45:03,704 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T06:45:03,746 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=43483, startcode=1733553901028 2024-12-07T06:45:03,749 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-07T06:45:03,751 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T06:45:03,752 WARN [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 400 ms and then retrying. 2024-12-07T06:45:03,784 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:03,789 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T06:45:03,790 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T06:45:03,791 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T06:45:03,810 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=33241, startcode=1733553901162 2024-12-07T06:45:03,812 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-07T06:45:03,814 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T06:45:03,814 WARN [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 400 ms and then retrying. 2024-12-07T06:45:03,876 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T06:45:03,877 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T06:45:03,879 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.large.0-1733553903878,5,FailOnTimeoutGroup] 2024-12-07T06:45:03,879 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T06:45:03,879 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:03,880 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.small.0-1733553903879,5,FailOnTimeoutGroup] 2024-12-07T06:45:03,880 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:03,881 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T06:45:03,882 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:03,914 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:04,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:45:04,118 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:45:04,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:04,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:04,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:45:04,137 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:45:04,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:04,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:04,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:45:04,168 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=43483, startcode=1733553901028 2024-12-07T06:45:04,172 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c02eafbb40,43483,1733553901028 2024-12-07T06:45:04,195 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] master.ServerManager(517): Registering regionserver=61c02eafbb40,43483,1733553901028 2024-12-07T06:45:04,217 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,35719,1733553900265 with port=33241, startcode=1733553901162 2024-12-07T06:45:04,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:45:04,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:04,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:04,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:45:04,229 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c02eafbb40,33241,1733553901162 2024-12-07T06:45:04,229 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35719 {}] master.ServerManager(517): Registering regionserver=61c02eafbb40,33241,1733553901162 2024-12-07T06:45:04,229 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:04,230 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:45:04,230 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:45:04,232 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:45:04,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:04,232 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:04,233 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:45:04,233 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:45:04,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:04,233 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:45:04,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:04,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:04,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:45:04,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:45:04,236 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:45:04,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:45:04,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:04,243 DEBUG [RS:1;61c02eafbb40:33241 {}] zookeeper.ZKUtil(111): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c02eafbb40,33241,1733553901162 2024-12-07T06:45:04,244 WARN [RS:1;61c02eafbb40:33241 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:45:04,244 INFO [RS:1;61c02eafbb40:33241 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:04,244 DEBUG [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,33241,1733553901162 2024-12-07T06:45:04,245 DEBUG [RS:0;61c02eafbb40:43483 {}] zookeeper.ZKUtil(111): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c02eafbb40,43483,1733553901028 2024-12-07T06:45:04,245 WARN [RS:0;61c02eafbb40:43483 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:45:04,246 INFO [RS:0;61c02eafbb40:43483 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:04,246 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,43483,1733553901028 2024-12-07T06:45:04,323 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c02eafbb40,33241,1733553901162] 2024-12-07T06:45:04,323 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c02eafbb40,43483,1733553901028] 2024-12-07T06:45:04,581 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:45:04,582 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75353255, jitterRate=0.12285099923610687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:45:04,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733553904051Initializing all the Stores at 1733553904051Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553904051Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553904060 (+9 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553904060Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553904060Cleaning up temporary data from old regions at 1733553904236 (+176 ms)Region opened successfully at 1733553904586 (+350 ms) 2024-12-07T06:45:04,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:45:04,587 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:45:04,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:45:04,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:45:04,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:45:04,587 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:45:04,671 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:45:04,672 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:45:04,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733553904586Disabling compacts and flushes for region at 1733553904586Disabling writes for close at 1733553904587 (+1 ms)Writing region close event to WAL at 1733553904671 (+84 ms)Closed at 1733553904671 2024-12-07T06:45:04,674 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:04,675 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T06:45:04,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T06:45:04,696 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:45:04,696 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:45:04,702 INFO [RS:0;61c02eafbb40:43483 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:45:04,702 INFO [RS:1;61c02eafbb40:33241 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:45:04,736 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,703 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,744 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:45:04,744 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:45:04,747 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:45:04,750 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T06:45:04,750 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:45:04,750 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:45:04,752 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,752 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,752 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,752 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,752 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,753 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c02eafbb40:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:45:04,754 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,754 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,754 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,754 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,755 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,755 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,755 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:04,755 DEBUG [RS:0;61c02eafbb40:43483 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:04,778 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c02eafbb40:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:45:04,778 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,779 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,779 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,779 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,779 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,780 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:04,780 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:04,780 DEBUG [RS:1;61c02eafbb40:33241 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:04,782 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,782 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,782 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,782 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,783 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,783 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,43483,1733553901028-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:45:04,845 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,846 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,846 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,847 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,847 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,847 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,33241,1733553901162-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:45:04,901 WARN [61c02eafbb40:35719 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T06:45:04,903 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:45:04,905 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,43483,1733553901028-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,905 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,905 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.Replication(171): 61c02eafbb40,43483,1733553901028 started 2024-12-07T06:45:04,947 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:45:04,948 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,33241,1733553901162-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,948 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,948 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.Replication(171): 61c02eafbb40,33241,1733553901162 started 2024-12-07T06:45:04,999 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,999 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:04,999 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1482): Serving as 61c02eafbb40,43483,1733553901028, RpcServer on 61c02eafbb40/172.17.0.2:43483, sessionid=0x1018bab34550001 2024-12-07T06:45:04,999 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(1482): Serving as 61c02eafbb40,33241,1733553901162, RpcServer on 61c02eafbb40/172.17.0.2:33241, sessionid=0x1018bab34550002 2024-12-07T06:45:05,000 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:45:05,000 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:45:05,000 DEBUG [RS:1;61c02eafbb40:33241 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c02eafbb40,33241,1733553901162 2024-12-07T06:45:05,001 DEBUG [RS:0;61c02eafbb40:43483 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c02eafbb40,43483,1733553901028 2024-12-07T06:45:05,001 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,33241,1733553901162' 2024-12-07T06:45:05,001 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,43483,1733553901028' 2024-12-07T06:45:05,001 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:45:05,001 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:45:05,002 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:45:05,002 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:45:05,003 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:45:05,003 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:45:05,003 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:45:05,003 DEBUG [RS:1;61c02eafbb40:33241 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c02eafbb40,33241,1733553901162 2024-12-07T06:45:05,003 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:45:05,003 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,33241,1733553901162' 2024-12-07T06:45:05,003 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:45:05,003 DEBUG [RS:0;61c02eafbb40:43483 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c02eafbb40,43483,1733553901028 2024-12-07T06:45:05,004 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,43483,1733553901028' 2024-12-07T06:45:05,004 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:45:05,004 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:45:05,004 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:45:05,005 DEBUG [RS:1;61c02eafbb40:33241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:45:05,005 INFO [RS:1;61c02eafbb40:33241 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:45:05,005 INFO [RS:1;61c02eafbb40:33241 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:45:05,005 DEBUG [RS:0;61c02eafbb40:43483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:45:05,005 INFO [RS:0;61c02eafbb40:43483 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:45:05,005 INFO [RS:0;61c02eafbb40:43483 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:45:05,111 INFO [RS:1;61c02eafbb40:33241 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:45:05,111 INFO [RS:0;61c02eafbb40:43483 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:45:05,113 INFO [RS:1;61c02eafbb40:33241 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c02eafbb40%2C33241%2C1733553901162, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,33241,1733553901162, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs, maxLogs=55 2024-12-07T06:45:05,113 INFO [RS:0;61c02eafbb40:43483 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c02eafbb40%2C43483%2C1733553901028, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,43483,1733553901028, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs, maxLogs=55 2024-12-07T06:45:05,145 INFO [RS:1;61c02eafbb40:33241 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,33241,1733553901162/61c02eafbb40%2C33241%2C1733553901162.1733553905121 2024-12-07T06:45:05,146 DEBUG [RS:1;61c02eafbb40:33241 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:05,146 INFO [RS:0;61c02eafbb40:43483 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,43483,1733553901028/61c02eafbb40%2C43483%2C1733553901028.1733553905121 2024-12-07T06:45:05,147 DEBUG [RS:0;61c02eafbb40:43483 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:05,404 DEBUG [61c02eafbb40:35719 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T06:45:05,416 DEBUG [61c02eafbb40:35719 {}] balancer.BalancerClusterState(204): Hosts are {61c02eafbb40=0} racks are {/default-rack=0} 2024-12-07T06:45:05,420 DEBUG [61c02eafbb40:35719 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:45:05,420 DEBUG [61c02eafbb40:35719 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:45:05,420 DEBUG [61c02eafbb40:35719 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:45:05,420 DEBUG [61c02eafbb40:35719 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:45:05,420 INFO [61c02eafbb40:35719 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:45:05,421 INFO [61c02eafbb40:35719 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:45:05,421 DEBUG [61c02eafbb40:35719 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:45:05,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c02eafbb40,43483,1733553901028 2024-12-07T06:45:05,435 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c02eafbb40,43483,1733553901028, state=OPENING 2024-12-07T06:45:05,441 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T06:45:05,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:05,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:05,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:05,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:05,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:05,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:05,447 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:45:05,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c02eafbb40,43483,1733553901028}] 2024-12-07T06:45:05,623 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:45:05,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50253, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:45:05,635 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T06:45:05,636 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:05,637 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T06:45:05,638 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c02eafbb40%2C43483%2C1733553901028.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,43483,1733553901028, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs, maxLogs=55 2024-12-07T06:45:05,661 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,43483,1733553901028/61c02eafbb40%2C43483%2C1733553901028.meta.1733553905640.meta 2024-12-07T06:45:05,662 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:05,662 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:05,664 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T06:45:05,667 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T06:45:05,672 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T06:45:05,677 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T06:45:05,677 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:05,678 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T06:45:05,678 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T06:45:05,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:45:05,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:45:05,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:05,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:05,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:45:05,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:45:05,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:05,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:05,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:45:05,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:45:05,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:05,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:05,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:45:05,685 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:45:05,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:05,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:05,686 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:45:05,687 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:05,688 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:05,689 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:45:05,689 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:45:05,689 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:45:05,690 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:45:05,691 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68389307, jitterRate=0.019080087542533875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:45:05,691 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T06:45:05,694 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733553905679Writing region info on filesystem at 1733553905679Initializing all the Stores at 1733553905679Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553905679Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553905680 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553905680Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553905680Cleaning up temporary data from old regions at 1733553905689 (+9 ms)Running coprocessor post-open hooks at 1733553905691 (+2 ms)Region opened successfully at 1733553905693 (+2 ms) 2024-12-07T06:45:05,700 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733553905614 2024-12-07T06:45:05,709 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T06:45:05,710 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T06:45:05,711 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c02eafbb40,43483,1733553901028 2024-12-07T06:45:05,712 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c02eafbb40,43483,1733553901028, state=OPEN 2024-12-07T06:45:05,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:45:05,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:45:05,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:45:05,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:05,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:05,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:05,715 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c02eafbb40,43483,1733553901028 2024-12-07T06:45:05,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T06:45:05,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c02eafbb40,43483,1733553901028 in 266 msec 2024-12-07T06:45:05,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T06:45:05,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0240 sec 2024-12-07T06:45:05,723 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:05,723 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T06:45:05,744 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:45:05,745 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c02eafbb40,43483,1733553901028, seqNum=-1] 2024-12-07T06:45:05,767 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:45:05,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:45:05,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 3.1050 sec 2024-12-07T06:45:05,794 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733553905794, completionTime=-1 2024-12-07T06:45:05,797 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T06:45:05,797 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T06:45:05,821 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T06:45:05,822 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733553965822 2024-12-07T06:45:05,822 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733554025822 2024-12-07T06:45:05,822 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-07T06:45:05,824 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T06:45:05,830 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35719,1733553900265-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:05,830 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35719,1733553900265-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:05,830 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35719,1733553900265-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:05,832 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c02eafbb40:35719, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:05,832 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:05,833 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:05,839 DEBUG [master/61c02eafbb40:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T06:45:05,860 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.599sec 2024-12-07T06:45:05,862 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T06:45:05,863 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T06:45:05,864 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T06:45:05,865 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T06:45:05,865 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T06:45:05,866 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35719,1733553900265-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:45:05,866 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35719,1733553900265-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T06:45:05,867 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T06:45:05,868 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T06:45:05,869 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35719,1733553900265-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:05,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:45:05,952 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T06:45:05,952 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T06:45:05,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c02eafbb40,35719,-1 for getting cluster id 2024-12-07T06:45:05,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T06:45:05,970 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'efb50fce-db46-4c2a-86da-374c5a8fffde' 2024-12-07T06:45:05,973 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T06:45:05,973 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "efb50fce-db46-4c2a-86da-374c5a8fffde" 2024-12-07T06:45:05,973 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceb0131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:45:05,973 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c02eafbb40,35719,-1] 2024-12-07T06:45:05,976 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T06:45:05,978 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:05,980 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T06:45:05,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:45:05,984 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:45:05,992 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c02eafbb40,43483,1733553901028, seqNum=-1] 2024-12-07T06:45:05,992 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:45:05,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:45:06,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c02eafbb40,35719,1733553900265 2024-12-07T06:45:06,035 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T06:45:06,040 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c02eafbb40,35719,1733553900265 2024-12-07T06:45:06,053 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@627ccf1a 2024-12-07T06:45:06,055 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T06:45:06,058 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T06:45:06,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733553906031', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:45:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 2024-12-07T06:45:06,073 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T06:45:06,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733553906031" procId is: 4 2024-12-07T06:45:06,076 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:06,077 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T06:45:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:06,097 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 26973d84386dcfbb97eddde33902aa2a, NAME => 'testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733553906031', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:06,114 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:06,115 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(1722): Closing 26973d84386dcfbb97eddde33902aa2a, disabling compactions & flushes 2024-12-07T06:45:06,115 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:06,115 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:06,115 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. after waiting 0 ms 2024-12-07T06:45:06,115 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:06,115 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:06,115 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733553906031-pool-0 {}] regionserver.HRegion(1676): Region close journal for 26973d84386dcfbb97eddde33902aa2a: Waiting for close lock at 1733553906114Disabling compacts and flushes for region at 1733553906114Disabling writes for close at 1733553906115 (+1 ms)Writing region close event to WAL at 1733553906115Closed at 1733553906115 2024-12-07T06:45:06,117 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T06:45:06,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733553906117"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733553906117"}]},"ts":"1733553906117"} 2024-12-07T06:45:06,129 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T06:45:06,131 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T06:45:06,134 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733553906031","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733553906131"}]},"ts":"1733553906131"} 2024-12-07T06:45:06,139 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733553906031, state=ENABLING in hbase:meta 2024-12-07T06:45:06,140 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {61c02eafbb40=0} racks are {/default-rack=0} 2024-12-07T06:45:06,141 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:45:06,141 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:45:06,141 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:45:06,141 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:45:06,141 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:45:06,141 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:45:06,141 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:45:06,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, ASSIGN}] 2024-12-07T06:45:06,145 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, ASSIGN 2024-12-07T06:45:06,152 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, ASSIGN; state=OFFLINE, location=61c02eafbb40,33241,1733553901162; forceNewPlan=false, retain=false 2024-12-07T06:45:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:06,304 INFO [61c02eafbb40:35719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T06:45:06,305 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=26973d84386dcfbb97eddde33902aa2a, regionState=OPENING, regionLocation=61c02eafbb40,33241,1733553901162 2024-12-07T06:45:06,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, ASSIGN because future has completed 2024-12-07T06:45:06,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,33241,1733553901162}] 2024-12-07T06:45:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:06,464 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:45:06,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33531, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:45:06,468 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:06,468 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 26973d84386dcfbb97eddde33902aa2a, NAME => 'testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:06,469 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733553906031 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,469 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:06,469 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,469 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,470 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,474 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26973d84386dcfbb97eddde33902aa2a columnFamilyName family 2024-12-07T06:45:06,474 DEBUG [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:06,475 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.HStore(327): Store=26973d84386dcfbb97eddde33902aa2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:06,475 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,475 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,476 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,476 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,476 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,477 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,483 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:45:06,483 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 26973d84386dcfbb97eddde33902aa2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73798035, jitterRate=0.09967641532421112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:45:06,483 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:06,484 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 26973d84386dcfbb97eddde33902aa2a: Running coprocessor pre-open hook at 1733553906469Writing region info on filesystem at 1733553906469Initializing all the Stores at 1733553906469Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553906470 (+1 ms)Cleaning up temporary data from old regions at 1733553906476 (+6 ms)Running coprocessor post-open hooks at 1733553906484 (+8 ms)Region opened successfully at 1733553906484 2024-12-07T06:45:06,486 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a., pid=6, masterSystemTime=1733553906463 2024-12-07T06:45:06,489 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:06,489 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:06,490 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=26973d84386dcfbb97eddde33902aa2a, regionState=OPEN, openSeqNum=2, regionLocation=61c02eafbb40,33241,1733553901162 2024-12-07T06:45:06,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,33241,1733553901162 because future has completed 2024-12-07T06:45:06,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T06:45:06,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,33241,1733553901162 in 183 msec 2024-12-07T06:45:06,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T06:45:06,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, ASSIGN in 353 msec 2024-12-07T06:45:06,499 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T06:45:06,499 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733553906031","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733553906499"}]},"ts":"1733553906499"} 2024-12-07T06:45:06,501 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733553906031, state=ENABLED in hbase:meta 2024-12-07T06:45:06,502 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T06:45:06,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733553906031 in 436 msec 2024-12-07T06:45:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:06,710 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733553906031 get assigned. Timeout = 60000ms 2024-12-07T06:45:06,710 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733553906031 completed 2024-12-07T06:45:06,713 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:45:06,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733553906031 assigned to meta. Checking AM states. 2024-12-07T06:45:06,721 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:45:06,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733553906031 assigned. 2024-12-07T06:45:06,730 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733553906031', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a., hostname=61c02eafbb40,33241,1733553901162, seqNum=2] 2024-12-07T06:45:06,732 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:45:06,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:45:06,745 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 26973d84386dcfbb97eddde33902aa2a 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T06:45:06,770 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:06,808 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/.tmp/family/a8b86a5807764fcd91d46dafa106dcdf is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T06:45:06,813 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/.tmp/family/a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:06,852 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/.tmp/family/a8b86a5807764fcd91d46dafa106dcdf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:06,854 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf, delay=946 ms 2024-12-07T06:45:06,856 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T06:45:06,862 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 26973d84386dcfbb97eddde33902aa2a in 116ms, sequenceid=6, compaction requested=false 2024-12-07T06:45:06,863 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733553906031' 2024-12-07T06:45:06,865 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 26973d84386dcfbb97eddde33902aa2a: 2024-12-07T06:45:07,802 DEBUG [hfile-prefetch-1733553906854 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8b86a5807764fcd91d46dafa106dcdf_0. 2024-12-07T06:45:07,806 DEBUG [hfile-prefetch-1733553906854 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8b86a5807764fcd91d46dafa106dcdf_117. 2024-12-07T06:45:07,807 DEBUG [hfile-prefetch-1733553906854 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf, with total blocks 2, and data blocks 1 2024-12-07T06:45:07,807 DEBUG [hfile-prefetch-1733553906854 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T06:45:07,807 DEBUG [hfile-prefetch-1733553906854 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T06:45:07,808 DEBUG [hfile-prefetch-1733553906854 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf. Will sleep 100ms and try the verification again. 2024-12-07T06:45:07,826 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T06:45:07,826 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T06:45:07,826 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:07,826 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:45:07,843 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:45:07,843 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733553907826 2024-12-07T06:45:07,843 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T06:45:07,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=26973d84386dcfbb97eddde33902aa2a, source=61c02eafbb40,33241,1733553901162, destination=61c02eafbb40,43483,1733553901028, warming up region on 61c02eafbb40,43483,1733553901028 2024-12-07T06:45:07,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=26973d84386dcfbb97eddde33902aa2a, source=61c02eafbb40,33241,1733553901162, destination=61c02eafbb40,43483,1733553901028, running balancer 2024-12-07T06:45:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, REOPEN/MOVE 2024-12-07T06:45:07,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:07,878 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, REOPEN/MOVE 2024-12-07T06:45:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(7855): Warmup {ENCODED => 26973d84386dcfbb97eddde33902aa2a, NAME => 'testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:07,879 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:07,880 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=26973d84386dcfbb97eddde33902aa2a, regionState=CLOSING, regionLocation=61c02eafbb40,33241,1733553901162 2024-12-07T06:45:07,880 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26973d84386dcfbb97eddde33902aa2a columnFamilyName family 2024-12-07T06:45:07,880 DEBUG [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:07,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, REOPEN/MOVE because future has completed 2024-12-07T06:45:07,883 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-07T06:45:07,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,33241,1733553901162}] 2024-12-07T06:45:07,884 DEBUG [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:07,884 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.HStore(327): Store=26973d84386dcfbb97eddde33902aa2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(1722): Closing 26973d84386dcfbb97eddde33902aa2a, disabling compactions & flushes 2024-12-07T06:45:07,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. after waiting 0 ms 2024-12-07T06:45:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:07,886 DEBUG [StoreFileCloser-26973d84386dcfbb97eddde33902aa2a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:07,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43483 {}] regionserver.HRegion(1676): Region close journal for 26973d84386dcfbb97eddde33902aa2a: Waiting for close lock at 1733553907884Disabling compacts and flushes for region at 1733553907884Disabling writes for close at 1733553907885 (+1 ms)Writing region close event to WAL at 1733553907886 (+1 ms)Closed at 1733553907886 2024-12-07T06:45:07,888 WARN [hfile-prefetch-1733553906854 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-07T06:45:07,889 DEBUG [hfile-prefetch-1733553906854 {}] hfile.PrefetchExecutor(121): Prefetch completed for a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35719 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-07T06:45:08,042 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,043 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-07T06:45:08,043 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 26973d84386dcfbb97eddde33902aa2a, disabling compactions & flushes 2024-12-07T06:45:08,043 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,043 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,043 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. after waiting 0 ms 2024-12-07T06:45:08,044 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,045 DEBUG [StoreFileCloser-26973d84386dcfbb97eddde33902aa2a-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file a8b86a5807764fcd91d46dafa106dcdf, starting offset: 0, end offset: 9223372036854775807 2024-12-07T06:45:08,046 DEBUG [StoreFileCloser-26973d84386dcfbb97eddde33902aa2a-family-1 {}] bucket.BucketCache(856): removed key a8b86a5807764fcd91d46dafa106dcdf_0 from back map with offset lock 2094080 in the evict process 2024-12-07T06:45:08,052 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T06:45:08,053 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,054 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 26973d84386dcfbb97eddde33902aa2a: Waiting for close lock at 1733553908043Running coprocessor pre-close hooks at 1733553908043Disabling compacts and flushes for region at 1733553908043Disabling writes for close at 1733553908044 (+1 ms)Writing region close event to WAL at 1733553908047 (+3 ms)Running coprocessor post-close hooks at 1733553908052 (+5 ms)Closed at 1733553908053 (+1 ms) 2024-12-07T06:45:08,054 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 26973d84386dcfbb97eddde33902aa2a move to 61c02eafbb40,43483,1733553901028 record at close sequenceid=6 2024-12-07T06:45:08,057 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,058 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=26973d84386dcfbb97eddde33902aa2a, regionState=CLOSED 2024-12-07T06:45:08,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,33241,1733553901162 because future has completed 2024-12-07T06:45:08,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T06:45:08,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,33241,1733553901162 in 178 msec 2024-12-07T06:45:08,064 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, REOPEN/MOVE; state=CLOSED, location=61c02eafbb40,43483,1733553901028; forceNewPlan=false, retain=false 2024-12-07T06:45:08,088 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T06:45:08,088 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T06:45:08,088 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:08,088 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:45:08,092 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:45:08,092 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733553908088 2024-12-07T06:45:08,092 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T06:45:08,215 INFO [61c02eafbb40:35719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T06:45:08,215 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=26973d84386dcfbb97eddde33902aa2a, regionState=OPENING, regionLocation=61c02eafbb40,43483,1733553901028 2024-12-07T06:45:08,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, REOPEN/MOVE because future has completed 2024-12-07T06:45:08,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,43483,1733553901028}] 2024-12-07T06:45:08,375 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,375 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 26973d84386dcfbb97eddde33902aa2a, NAME => 'testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:08,375 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733553906031 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,375 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:08,376 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,376 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,376 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,378 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26973d84386dcfbb97eddde33902aa2a columnFamilyName family 2024-12-07T06:45:08,378 DEBUG [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:08,379 DEBUG [StoreFileOpener-26973d84386dcfbb97eddde33902aa2a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf, delay=975 ms 2024-12-07T06:45:08,381 DEBUG [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:08,381 INFO [StoreOpener-26973d84386dcfbb97eddde33902aa2a-1 {}] regionserver.HStore(327): Store=26973d84386dcfbb97eddde33902aa2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:08,381 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,382 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,382 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,382 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,382 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,387 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,387 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 26973d84386dcfbb97eddde33902aa2a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70879274, jitterRate=0.05618348717689514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:45:08,387 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,388 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 26973d84386dcfbb97eddde33902aa2a: Running coprocessor pre-open hook at 1733553908376Writing region info on filesystem at 1733553908376Initializing all the Stores at 1733553908376Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553908376Cleaning up temporary data from old regions at 1733553908382 (+6 ms)Running coprocessor post-open hooks at 1733553908387 (+5 ms)Region opened successfully at 1733553908388 (+1 ms) 2024-12-07T06:45:08,389 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a., pid=9, masterSystemTime=1733553908372 2024-12-07T06:45:08,391 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,391 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=26973d84386dcfbb97eddde33902aa2a, regionState=OPEN, openSeqNum=10, regionLocation=61c02eafbb40,43483,1733553901028 2024-12-07T06:45:08,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,43483,1733553901028 because future has completed 2024-12-07T06:45:08,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-07T06:45:08,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 26973d84386dcfbb97eddde33902aa2a, server=61c02eafbb40,43483,1733553901028 in 177 msec 2024-12-07T06:45:08,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733553906031, region=26973d84386dcfbb97eddde33902aa2a, REOPEN/MOVE in 521 msec 2024-12-07T06:45:08,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T06:45:08,431 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:45:08,431 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:08,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:08,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:08,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T06:45:08,436 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T06:45:08,436 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-12-07T06:45:08,437 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c02eafbb40,35719,1733553900265 2024-12-07T06:45:08,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:08,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:08,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:08,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:08,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:08,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:08,439 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:45:08,440 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:45:08,440 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:08,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:08,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:08,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:08,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:08,440 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c02eafbb40,43483,1733553901028' ***** 2024-12-07T06:45:08,440 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:45:08,440 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c02eafbb40,33241,1733553901162' ***** 2024-12-07T06:45:08,441 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:45:08,441 INFO [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:45:08,441 INFO [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.persistence 2024-12-07T06:45:08,441 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:45:08,441 INFO [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:45:08,441 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:45:08,441 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:45:08,441 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:45:08,442 INFO [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/bucket.persistence 2024-12-07T06:45:08,442 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:45:08,442 INFO [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:45:08,442 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:45:08,442 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:45:08,442 INFO [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:45:08,442 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:45:08,452 DEBUG [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1487): Thread RS:0;61c02eafbb40:43483 started persisting bucket cache to file 2024-12-07T06:45:08,452 DEBUG [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:08,452 DEBUG [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:45:08,452 DEBUG [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1487): Thread RS:1;61c02eafbb40:33241 started persisting bucket cache to file 2024-12-07T06:45:08,452 DEBUG [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:08,452 DEBUG [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:45:08,454 INFO [regionserver/61c02eafbb40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:08,455 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:45:08,456 DEBUG [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:45:08,456 DEBUG [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733553908452 2024-12-07T06:45:08,456 DEBUG [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:45:08,456 DEBUG [RS:1;61c02eafbb40:33241 {}] bucket.BucketCache(1509): Thread RS:1;61c02eafbb40:33241 finished persisting bucket cache to file, renaming 2024-12-07T06:45:08,456 DEBUG [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733553908452 2024-12-07T06:45:08,456 DEBUG [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1509): Thread RS:0;61c02eafbb40:43483 finished persisting bucket cache to file, renaming 2024-12-07T06:45:08,456 WARN [RS:0;61c02eafbb40:43483 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-07T06:45:08,456 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:45:08,456 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:45:08,456 INFO [RS:1;61c02eafbb40:33241 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:45:08,456 INFO [RS:0;61c02eafbb40:43483 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:45:08,456 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:45:08,457 INFO [RS:1;61c02eafbb40:33241 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:45:08,457 INFO [RS:0;61c02eafbb40:43483 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:45:08,457 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(959): stopping server 61c02eafbb40,33241,1733553901162 2024-12-07T06:45:08,457 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(3091): Received CLOSE for 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,457 INFO [RS:1;61c02eafbb40:33241 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:45:08,457 INFO [RS:1;61c02eafbb40:33241 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;61c02eafbb40:33241. 2024-12-07T06:45:08,457 DEBUG [RS:1;61c02eafbb40:33241 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:08,457 DEBUG [RS:1;61c02eafbb40:33241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:08,457 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(976): stopping server 61c02eafbb40,33241,1733553901162; all regions closed. 2024-12-07T06:45:08,457 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(959): stopping server 61c02eafbb40,43483,1733553901028 2024-12-07T06:45:08,457 INFO [RS:0;61c02eafbb40:43483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:45:08,457 INFO [RS:0;61c02eafbb40:43483 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c02eafbb40:43483. 2024-12-07T06:45:08,458 DEBUG [RS:0;61c02eafbb40:43483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:08,458 DEBUG [RS:0;61c02eafbb40:43483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:08,458 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:45:08,458 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:45:08,458 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:45:08,458 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:45:08,458 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T06:45:08,458 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 26973d84386dcfbb97eddde33902aa2a, disabling compactions & flushes 2024-12-07T06:45:08,458 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,458 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,458 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T06:45:08,459 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. after waiting 0 ms 2024-12-07T06:45:08,459 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 26973d84386dcfbb97eddde33902aa2a=testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a.} 2024-12-07T06:45:08,459 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:45:08,459 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,459 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:45:08,459 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:45:08,459 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:45:08,459 DEBUG [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26973d84386dcfbb97eddde33902aa2a 2024-12-07T06:45:08,459 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:45:08,459 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:45:08,459 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-07T06:45:08,459 DEBUG [StoreFileCloser-26973d84386dcfbb97eddde33902aa2a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/family/a8b86a5807764fcd91d46dafa106dcdf 2024-12-07T06:45:08,463 DEBUG [RS:1;61c02eafbb40:33241 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs 2024-12-07T06:45:08,464 INFO [RS:1;61c02eafbb40:33241 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c02eafbb40%2C33241%2C1733553901162:(num 1733553905121) 2024-12-07T06:45:08,464 DEBUG [RS:1;61c02eafbb40:33241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:08,464 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:08,464 INFO [RS:1;61c02eafbb40:33241 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:45:08,464 INFO [RS:1;61c02eafbb40:33241 {}] hbase.ChoreService(370): Chore service for: regionserver/61c02eafbb40:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T06:45:08,464 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:45:08,464 INFO [regionserver/61c02eafbb40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:45:08,464 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:45:08,464 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnRegionMove1733553906031/26973d84386dcfbb97eddde33902aa2a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T06:45:08,465 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:45:08,465 INFO [RS:1;61c02eafbb40:33241 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:45:08,465 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,465 INFO [RS:1;61c02eafbb40:33241 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33241 2024-12-07T06:45:08,465 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 26973d84386dcfbb97eddde33902aa2a: Waiting for close lock at 1733553908458Running coprocessor pre-close hooks at 1733553908458Disabling compacts and flushes for region at 1733553908458Disabling writes for close at 1733553908459 (+1 ms)Writing region close event to WAL at 1733553908460 (+1 ms)Running coprocessor post-close hooks at 1733553908465 (+5 ms)Closed at 1733553908465 2024-12-07T06:45:08,465 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a. 2024-12-07T06:45:08,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c02eafbb40,33241,1733553901162 2024-12-07T06:45:08,469 INFO [RS:1;61c02eafbb40:33241 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:45:08,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:08,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c02eafbb40,33241,1733553901162] 2024-12-07T06:45:08,474 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1b2e8fbacb9e4c639a6e654dc549ec53 2024-12-07T06:45:08,474 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c02eafbb40,33241,1733553901162 already deleted, retry=false 2024-12-07T06:45:08,474 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c02eafbb40,33241,1733553901162 expired; onlineServers=1 2024-12-07T06:45:08,490 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/info/1b2e8fbacb9e4c639a6e654dc549ec53 is 205, key is testBlockEvictionOnRegionMove1733553906031,,1733553906059.26973d84386dcfbb97eddde33902aa2a./info:regioninfo/1733553908391/Put/seqid=0 2024-12-07T06:45:08,491 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/info/1b2e8fbacb9e4c639a6e654dc549ec53 2024-12-07T06:45:08,499 INFO [regionserver/61c02eafbb40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:08,501 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 41d9a0b5563c49e7b655200d8fcf4da1 2024-12-07T06:45:08,514 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/ns/41d9a0b5563c49e7b655200d8fcf4da1 is 43, key is default/ns:d/1733553905774/Put/seqid=0 2024-12-07T06:45:08,514 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/ns/41d9a0b5563c49e7b655200d8fcf4da1 2024-12-07T06:45:08,523 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46f050542a28464aabf2b3fa80121ea9 2024-12-07T06:45:08,537 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/table/46f050542a28464aabf2b3fa80121ea9 is 78, key is testBlockEvictionOnRegionMove1733553906031/table:state/1733553906499/Put/seqid=0 2024-12-07T06:45:08,538 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/table/46f050542a28464aabf2b3fa80121ea9 2024-12-07T06:45:08,541 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/info/1b2e8fbacb9e4c639a6e654dc549ec53 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/1b2e8fbacb9e4c639a6e654dc549ec53 2024-12-07T06:45:08,542 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/1b2e8fbacb9e4c639a6e654dc549ec53, delay=960 ms 2024-12-07T06:45:08,543 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/1b2e8fbacb9e4c639a6e654dc549ec53, entries=15, sequenceid=15, filesize=7.6 K 2024-12-07T06:45:08,544 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/ns/41d9a0b5563c49e7b655200d8fcf4da1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/41d9a0b5563c49e7b655200d8fcf4da1 2024-12-07T06:45:08,545 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/41d9a0b5563c49e7b655200d8fcf4da1, delay=945 ms 2024-12-07T06:45:08,546 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/41d9a0b5563c49e7b655200d8fcf4da1, entries=2, sequenceid=15, filesize=5.0 K 2024-12-07T06:45:08,546 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/table/46f050542a28464aabf2b3fa80121ea9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46f050542a28464aabf2b3fa80121ea9 2024-12-07T06:45:08,548 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46f050542a28464aabf2b3fa80121ea9, delay=960 ms 2024-12-07T06:45:08,549 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46f050542a28464aabf2b3fa80121ea9, entries=2, sequenceid=15, filesize=5.3 K 2024-12-07T06:45:08,550 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false 2024-12-07T06:45:08,550 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T06:45:08,551 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/1b2e8fbacb9e4c639a6e654dc549ec53 2024-12-07T06:45:08,552 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/41d9a0b5563c49e7b655200d8fcf4da1 2024-12-07T06:45:08,553 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46f050542a28464aabf2b3fa80121ea9 2024-12-07T06:45:08,557 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-07T06:45:08,557 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T06:45:08,558 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:45:08,558 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733553908459Running coprocessor pre-close hooks at 1733553908459Disabling compacts and flushes for region at 1733553908459Disabling writes for close at 1733553908459Obtaining lock to block concurrent updates at 1733553908459Preparing flush snapshotting stores in 1588230740 at 1733553908459Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733553908460 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733553908460Flushing 1588230740/info: creating writer at 1733553908461 (+1 ms)Flushing 1588230740/info: appending metadata at 1733553908490 (+29 ms)Flushing 1588230740/info: closing flushed file at 1733553908490Flushing 1588230740/ns: creating writer at 1733553908494 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733553908513 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733553908513Flushing 1588230740/table: creating writer at 1733553908517 (+4 ms)Flushing 1588230740/table: appending metadata at 1733553908537 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733553908537Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17fd6aef: reopening flushed file at 1733553908540 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ccd7b09: reopening flushed file at 1733553908543 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3183c55e: reopening flushed file at 1733553908546 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false at 1733553908550 (+4 ms)Writing region close event to WAL at 1733553908553 (+3 ms)Running coprocessor post-close hooks at 1733553908557 (+4 ms)Closed at 1733553908558 (+1 ms) 2024-12-07T06:45:08,558 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T06:45:08,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:08,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33241-0x1018bab34550002, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:08,572 INFO [RS:1;61c02eafbb40:33241 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:45:08,572 INFO [RS:1;61c02eafbb40:33241 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c02eafbb40,33241,1733553901162; zookeeper connection closed. 2024-12-07T06:45:08,573 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1f4e2fd4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1f4e2fd4 2024-12-07T06:45:08,659 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(976): stopping server 61c02eafbb40,43483,1733553901028; all regions closed. 2024-12-07T06:45:08,662 DEBUG [RS:0;61c02eafbb40:43483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs 2024-12-07T06:45:08,662 INFO [RS:0;61c02eafbb40:43483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c02eafbb40%2C43483%2C1733553901028.meta:.meta(num 1733553905640) 2024-12-07T06:45:08,664 DEBUG [RS:0;61c02eafbb40:43483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs 2024-12-07T06:45:08,664 INFO [RS:0;61c02eafbb40:43483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c02eafbb40%2C43483%2C1733553901028:(num 1733553905121) 2024-12-07T06:45:08,664 DEBUG [RS:0;61c02eafbb40:43483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:08,664 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:08,664 INFO [RS:0;61c02eafbb40:43483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:45:08,664 INFO [RS:0;61c02eafbb40:43483 {}] hbase.ChoreService(370): Chore service for: regionserver/61c02eafbb40:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T06:45:08,664 INFO [RS:0;61c02eafbb40:43483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:45:08,664 INFO [regionserver/61c02eafbb40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:45:08,665 INFO [RS:0;61c02eafbb40:43483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43483 2024-12-07T06:45:08,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c02eafbb40,43483,1733553901028 2024-12-07T06:45:08,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:08,667 INFO [RS:0;61c02eafbb40:43483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:45:08,668 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c02eafbb40,43483,1733553901028] 2024-12-07T06:45:08,670 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c02eafbb40,43483,1733553901028 already deleted, retry=false 2024-12-07T06:45:08,670 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c02eafbb40,43483,1733553901028 expired; onlineServers=0 2024-12-07T06:45:08,670 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c02eafbb40,35719,1733553900265' ***** 2024-12-07T06:45:08,670 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T06:45:08,670 INFO [M:0;61c02eafbb40:35719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:45:08,670 INFO [M:0;61c02eafbb40:35719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:45:08,671 DEBUG [M:0;61c02eafbb40:35719 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T06:45:08,671 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T06:45:08,671 DEBUG [M:0;61c02eafbb40:35719 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T06:45:08,671 DEBUG [master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.small.0-1733553903879 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.small.0-1733553903879,5,FailOnTimeoutGroup] 2024-12-07T06:45:08,671 DEBUG [master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.large.0-1733553903878 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.large.0-1733553903878,5,FailOnTimeoutGroup] 2024-12-07T06:45:08,671 INFO [M:0;61c02eafbb40:35719 {}] hbase.ChoreService(370): Chore service for: master/61c02eafbb40:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T06:45:08,671 INFO [M:0;61c02eafbb40:35719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:45:08,671 DEBUG [M:0;61c02eafbb40:35719 {}] master.HMaster(1795): Stopping service threads 2024-12-07T06:45:08,671 INFO [M:0;61c02eafbb40:35719 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T06:45:08,672 INFO [M:0;61c02eafbb40:35719 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:45:08,672 INFO [M:0;61c02eafbb40:35719 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T06:45:08,672 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T06:45:08,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T06:45:08,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:08,675 DEBUG [M:0;61c02eafbb40:35719 {}] zookeeper.ZKUtil(347): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T06:45:08,675 WARN [M:0;61c02eafbb40:35719 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T06:45:08,675 INFO [M:0;61c02eafbb40:35719 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.lastflushedseqids 2024-12-07T06:45:08,690 INFO [M:0;61c02eafbb40:35719 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T06:45:08,690 INFO [M:0;61c02eafbb40:35719 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T06:45:08,690 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:45:08,690 INFO [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:08,690 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:08,690 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:45:08,690 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:08,691 INFO [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-07T06:45:08,701 DEBUG [M:0;61c02eafbb40:35719 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e48b8123fbf948a882801e9cd2d77134 2024-12-07T06:45:08,721 DEBUG [M:0;61c02eafbb40:35719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e48b8123fbf948a882801e9cd2d77134 is 82, key is hbase:meta,,1/info:regioninfo/1733553905710/Put/seqid=0 2024-12-07T06:45:08,722 INFO [M:0;61c02eafbb40:35719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e48b8123fbf948a882801e9cd2d77134 2024-12-07T06:45:08,731 DEBUG [M:0;61c02eafbb40:35719 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9d8d8ef6a1c54be382f0b4aa7887e8e3 2024-12-07T06:45:08,744 DEBUG [M:0;61c02eafbb40:35719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9d8d8ef6a1c54be382f0b4aa7887e8e3 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733553906504/Put/seqid=0 2024-12-07T06:45:08,745 INFO [M:0;61c02eafbb40:35719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9d8d8ef6a1c54be382f0b4aa7887e8e3 2024-12-07T06:45:08,753 DEBUG [M:0;61c02eafbb40:35719 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5b7015990c60414683b0de48fff1a96c 2024-12-07T06:45:08,766 DEBUG [M:0;61c02eafbb40:35719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5b7015990c60414683b0de48fff1a96c is 69, key is 61c02eafbb40,33241,1733553901162/rs:state/1733553904229/Put/seqid=0 2024-12-07T06:45:08,767 INFO [M:0;61c02eafbb40:35719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5b7015990c60414683b0de48fff1a96c 2024-12-07T06:45:08,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:08,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43483-0x1018bab34550001, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:08,769 INFO [RS:0;61c02eafbb40:43483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:45:08,769 INFO [RS:0;61c02eafbb40:43483 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c02eafbb40,43483,1733553901028; zookeeper connection closed. 2024-12-07T06:45:08,769 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e48b8123fbf948a882801e9cd2d77134 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e48b8123fbf948a882801e9cd2d77134 2024-12-07T06:45:08,769 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@76a62f7b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@76a62f7b 2024-12-07T06:45:08,770 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-07T06:45:08,771 INFO [M:0;61c02eafbb40:35719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e48b8123fbf948a882801e9cd2d77134, entries=8, sequenceid=84, filesize=5.5 K 2024-12-07T06:45:08,771 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9d8d8ef6a1c54be382f0b4aa7887e8e3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9d8d8ef6a1c54be382f0b4aa7887e8e3 2024-12-07T06:45:08,773 INFO [M:0;61c02eafbb40:35719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9d8d8ef6a1c54be382f0b4aa7887e8e3, entries=9, sequenceid=84, filesize=6.5 K 2024-12-07T06:45:08,773 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5b7015990c60414683b0de48fff1a96c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5b7015990c60414683b0de48fff1a96c 2024-12-07T06:45:08,775 INFO [M:0;61c02eafbb40:35719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5b7015990c60414683b0de48fff1a96c, entries=2, sequenceid=84, filesize=5.1 K 2024-12-07T06:45:08,776 INFO [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false 2024-12-07T06:45:08,777 INFO [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:08,778 DEBUG [M:0;61c02eafbb40:35719 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733553908690Disabling compacts and flushes for region at 1733553908690Disabling writes for close at 1733553908690Obtaining lock to block concurrent updates at 1733553908691 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733553908691Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733553908691Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733553908692 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733553908692Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733553908721 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733553908721Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733553908725 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733553908744 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733553908744Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733553908747 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733553908766 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733553908766Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7955e4fb: reopening flushed file at 1733553908769 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c030262: reopening flushed file at 1733553908771 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c5d24a5: reopening flushed file at 1733553908773 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false at 1733553908776 (+3 ms)Writing region close event to WAL at 1733553908777 (+1 ms)Closed at 1733553908777 2024-12-07T06:45:08,779 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:45:08,779 INFO [M:0;61c02eafbb40:35719 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T06:45:08,779 INFO [M:0;61c02eafbb40:35719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35719 2024-12-07T06:45:08,780 INFO [M:0;61c02eafbb40:35719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:45:08,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:08,883 INFO [M:0;61c02eafbb40:35719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:45:08,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35719-0x1018bab34550000, quorum=127.0.0.1:60383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:08,888 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T06:45:08,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T06:45:08,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:08,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/61c02eafbb40:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/61c02eafbb40:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733553908379 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733553908545 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/61c02eafbb40:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/61c02eafbb40:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1733553908542 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733553906854 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 370) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5605 (was 6181) 2024-12-07T06:45:08,925 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=5604 2024-12-07T06:45:08,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/hadoop-log-dir so I do NOT create it in target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c 2024-12-07T06:45:08,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/hadoop-tmp-dir so I do NOT create it in target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c 2024-12-07T06:45:08,926 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:08,929 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/cluster_d9dd91b2-f88a-9ac3-7c5b-4aef105d11d9, deleteOnExit=true 2024-12-07T06:45:08,932 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/cluster_d9dd91b2-f88a-9ac3-7c5b-4aef105d11d9/zookeeper_0, clientPort=51898, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/cluster_d9dd91b2-f88a-9ac3-7c5b-4aef105d11d9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/cluster_d9dd91b2-f88a-9ac3-7c5b-4aef105d11d9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T06:45:08,933 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51898 2024-12-07T06:45:08,944 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c with version=8 2024-12-07T06:45:08,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/hbase-staging 2024-12-07T06:45:08,947 INFO [Time-limited test {}] client.ConnectionUtils(128): master/61c02eafbb40:0 server-side Connection retries=45 2024-12-07T06:45:08,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:08,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:08,947 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:45:08,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:08,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:45:08,948 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T06:45:08,948 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:45:08,948 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44987 2024-12-07T06:45:08,950 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44987 connecting to ZooKeeper ensemble=127.0.0.1:51898 2024-12-07T06:45:08,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449870x0, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:45:08,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44987-0x1018bab57530000 connected 2024-12-07T06:45:08,976 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:08,977 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:08,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:08,977 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c, hbase.cluster.distributed=false 2024-12-07T06:45:08,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:45:08,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44987 2024-12-07T06:45:08,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44987 2024-12-07T06:45:08,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44987 2024-12-07T06:45:08,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44987 2024-12-07T06:45:08,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44987 2024-12-07T06:45:08,998 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c02eafbb40:0 server-side Connection retries=45 2024-12-07T06:45:08,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:08,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:08,998 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:45:08,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:08,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:45:08,998 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:45:08,999 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:45:08,999 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37853 2024-12-07T06:45:09,001 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37853 connecting to ZooKeeper ensemble=127.0.0.1:51898 2024-12-07T06:45:09,002 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:09,002 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:09,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378530x0, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:45:09,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378530x0, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:09,005 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37853-0x1018bab57530001 connected 2024-12-07T06:45:09,006 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:45:09,006 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.cache 2024-12-07T06:45:09,006 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:45:09,007 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:45:09,007 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:45:09,007 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:45:09,007 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:45:09,007 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:45:09,008 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:45:09,008 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:45:09,008 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence is complete. 2024-12-07T06:45:09,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:45:09,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:45:09,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37853 2024-12-07T06:45:09,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37853 2024-12-07T06:45:09,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37853 2024-12-07T06:45:09,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37853 2024-12-07T06:45:09,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37853 2024-12-07T06:45:09,036 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c02eafbb40:0 server-side Connection retries=45 2024-12-07T06:45:09,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:09,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:09,036 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:45:09,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:09,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:45:09,036 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:45:09,037 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:45:09,037 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38247 2024-12-07T06:45:09,039 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38247 connecting to ZooKeeper ensemble=127.0.0.1:51898 2024-12-07T06:45:09,039 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:09,039 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:09,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382470x0, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:45:09,043 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38247-0x1018bab57530002 connected 2024-12-07T06:45:09,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:09,043 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:45:09,044 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.cache 2024-12-07T06:45:09,044 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:45:09,044 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:45:09,044 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:45:09,046 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:45:09,046 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:45:09,046 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:45:09,047 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:45:09,047 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:45:09,047 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence is complete. 2024-12-07T06:45:09,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:45:09,049 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:45:09,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38247 2024-12-07T06:45:09,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38247 2024-12-07T06:45:09,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38247 2024-12-07T06:45:09,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38247 2024-12-07T06:45:09,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38247 2024-12-07T06:45:09,068 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c02eafbb40:44987 2024-12-07T06:45:09,069 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c02eafbb40,44987,1733553908946 2024-12-07T06:45:09,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:09,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:09,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:09,072 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c02eafbb40,44987,1733553908946 2024-12-07T06:45:09,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:45:09,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:45:09,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,077 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T06:45:09,077 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c02eafbb40,44987,1733553908946 from backup master directory 2024-12-07T06:45:09,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:09,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:09,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c02eafbb40,44987,1733553908946 2024-12-07T06:45:09,079 WARN [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:45:09,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:45:09,079 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c02eafbb40,44987,1733553908946 2024-12-07T06:45:09,080 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/hbase.id] with ID: 7c08351d-3ba5-4c0a-a882-7f77fcc5b4e8 2024-12-07T06:45:09,080 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.tmp/hbase.id 2024-12-07T06:45:09,087 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/hbase.id] 2024-12-07T06:45:09,130 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:09,130 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T06:45:09,132 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-07T06:45:09,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,140 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:45:09,141 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T06:45:09,142 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:09,163 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store 2024-12-07T06:45:09,170 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:09,170 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:45:09,170 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:09,170 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:09,170 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:45:09,170 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:09,170 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:09,170 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733553909170Disabling compacts and flushes for region at 1733553909170Disabling writes for close at 1733553909170Writing region close event to WAL at 1733553909170Closed at 1733553909170 2024-12-07T06:45:09,170 WARN [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/.initializing 2024-12-07T06:45:09,170 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/WALs/61c02eafbb40,44987,1733553908946 2024-12-07T06:45:09,176 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:45:09,179 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c02eafbb40%2C44987%2C1733553908946, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/WALs/61c02eafbb40,44987,1733553908946, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/oldWALs, maxLogs=10 2024-12-07T06:45:09,197 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/WALs/61c02eafbb40,44987,1733553908946/61c02eafbb40%2C44987%2C1733553908946.1733553909179 2024-12-07T06:45:09,198 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:09,198 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:09,198 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:09,198 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,198 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,199 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T06:45:09,203 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T06:45:09,208 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:09,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T06:45:09,211 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:09,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T06:45:09,215 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:09,215 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,216 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,216 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,216 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,216 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,216 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:45:09,217 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:45:09,223 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:45:09,223 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63212828, jitterRate=-0.05805546045303345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:45:09,224 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733553909198Initializing all the Stores at 1733553909199 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553909199Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553909199Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553909199Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553909199Cleaning up temporary data from old regions at 1733553909216 (+17 ms)Region opened successfully at 1733553909224 (+8 ms) 2024-12-07T06:45:09,225 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T06:45:09,228 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b0e029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c02eafbb40/172.17.0.2:0 2024-12-07T06:45:09,230 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T06:45:09,230 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T06:45:09,230 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T06:45:09,230 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T06:45:09,230 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T06:45:09,231 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-07T06:45:09,231 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T06:45:09,232 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T06:45:09,233 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T06:45:09,235 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T06:45:09,235 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T06:45:09,236 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T06:45:09,238 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T06:45:09,238 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T06:45:09,239 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T06:45:09,241 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T06:45:09,242 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T06:45:09,243 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T06:45:09,245 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T06:45:09,247 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T06:45:09,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:09,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:09,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:09,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,251 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c02eafbb40,44987,1733553908946, sessionid=0x1018bab57530000, setting cluster-up flag (Was=false) 2024-12-07T06:45:09,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,260 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T06:45:09,261 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c02eafbb40,44987,1733553908946 2024-12-07T06:45:09,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,269 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T06:45:09,270 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c02eafbb40,44987,1733553908946 2024-12-07T06:45:09,271 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T06:45:09,272 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:09,272 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T06:45:09,272 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T06:45:09,273 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c02eafbb40,44987,1733553908946 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c02eafbb40:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c02eafbb40:0, corePoolSize=10, maxPoolSize=10 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c02eafbb40:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:45:09,274 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733553939276 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T06:45:09,276 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:09,276 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,276 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T06:45:09,277 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T06:45:09,277 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T06:45:09,277 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,277 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T06:45:09,277 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T06:45:09,281 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T06:45:09,281 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T06:45:09,281 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.large.0-1733553909281,5,FailOnTimeoutGroup] 2024-12-07T06:45:09,283 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.small.0-1733553909281,5,FailOnTimeoutGroup] 2024-12-07T06:45:09,283 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,283 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T06:45:09,283 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,283 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,292 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T06:45:09,293 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:09,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:09,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:45:09,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:45:09,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:45:09,306 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:45:09,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:45:09,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:45:09,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:45:09,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:45:09,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:45:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:09,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:45:09,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:45:09,315 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:45:09,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:45:09,321 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:45:09,321 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60330158, jitterRate=-0.10101059079170227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:45:09,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733553909299Initializing all the Stores at 1733553909299Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553909299Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553909299Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553909299Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553909299Cleaning up temporary data from old regions at 1733553909315 (+16 ms)Region opened successfully at 1733553909322 (+7 ms) 2024-12-07T06:45:09,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:45:09,323 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:45:09,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:45:09,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:45:09,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:45:09,323 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:45:09,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733553909322Disabling compacts and flushes for region at 1733553909322Disabling writes for close at 1733553909323 (+1 ms)Writing region close event to WAL at 1733553909323Closed at 1733553909323 2024-12-07T06:45:09,324 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:09,324 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T06:45:09,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T06:45:09,325 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:45:09,326 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T06:45:09,359 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(746): ClusterId : 7c08351d-3ba5-4c0a-a882-7f77fcc5b4e8 2024-12-07T06:45:09,359 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(746): ClusterId : 7c08351d-3ba5-4c0a-a882-7f77fcc5b4e8 2024-12-07T06:45:09,359 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:45:09,359 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:45:09,361 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:45:09,361 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:45:09,361 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:45:09,361 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:45:09,365 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:45:09,365 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:45:09,365 DEBUG [RS:1;61c02eafbb40:38247 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349e2c07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c02eafbb40/172.17.0.2:0 2024-12-07T06:45:09,366 DEBUG [RS:0;61c02eafbb40:37853 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36cb0b67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c02eafbb40/172.17.0.2:0 2024-12-07T06:45:09,379 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;61c02eafbb40:38247 2024-12-07T06:45:09,379 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:45:09,379 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:45:09,379 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:45:09,380 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,44987,1733553908946 with port=38247, startcode=1733553909035 2024-12-07T06:45:09,381 DEBUG [RS:1;61c02eafbb40:38247 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:45:09,382 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c02eafbb40:37853 2024-12-07T06:45:09,382 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:45:09,382 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:45:09,382 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:45:09,383 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,44987,1733553908946 with port=37853, startcode=1733553908997 2024-12-07T06:45:09,383 DEBUG [RS:0;61c02eafbb40:37853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:45:09,383 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55075, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:45:09,384 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c02eafbb40,38247,1733553909035 2024-12-07T06:45:09,384 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44987 {}] master.ServerManager(517): Registering regionserver=61c02eafbb40,38247,1733553909035 2024-12-07T06:45:09,385 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:45:09,386 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,386 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44987 {}] master.ServerManager(517): Registering regionserver=61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,386 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:09,386 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:45:09,386 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:45:09,387 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:09,387 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:45:09,387 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:45:09,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:09,392 DEBUG [RS:1;61c02eafbb40:38247 {}] zookeeper.ZKUtil(111): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c02eafbb40,38247,1733553909035 2024-12-07T06:45:09,392 WARN [RS:1;61c02eafbb40:38247 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:45:09,392 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c02eafbb40,38247,1733553909035] 2024-12-07T06:45:09,392 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c02eafbb40,37853,1733553908997] 2024-12-07T06:45:09,392 INFO [RS:1;61c02eafbb40:38247 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:09,392 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,38247,1733553909035 2024-12-07T06:45:09,392 DEBUG [RS:0;61c02eafbb40:37853 {}] zookeeper.ZKUtil(111): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,393 WARN [RS:0;61c02eafbb40:37853 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:45:09,393 INFO [RS:0;61c02eafbb40:37853 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:09,393 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,398 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:45:09,402 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:45:09,407 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:45:09,407 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:45:09,408 INFO [RS:1;61c02eafbb40:38247 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:45:09,408 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,408 INFO [RS:0;61c02eafbb40:37853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:45:09,408 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,415 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:45:09,415 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:45:09,416 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:45:09,416 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:45:09,416 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,416 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,416 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,416 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,416 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,416 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,416 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c02eafbb40:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c02eafbb40:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,417 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,418 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:09,418 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,418 DEBUG [RS:1;61c02eafbb40:38247 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:09,418 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:09,418 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:09,418 DEBUG [RS:0;61c02eafbb40:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:09,418 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,418 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,38247,1733553909035-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,419 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,37853,1733553908997-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:45:09,435 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:45:09,436 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,38247,1733553909035-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,436 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:45:09,436 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,436 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,37853,1733553908997-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,436 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.Replication(171): 61c02eafbb40,38247,1733553909035 started 2024-12-07T06:45:09,436 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,436 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.Replication(171): 61c02eafbb40,37853,1733553908997 started 2024-12-07T06:45:09,450 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,450 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1482): Serving as 61c02eafbb40,37853,1733553908997, RpcServer on 61c02eafbb40/172.17.0.2:37853, sessionid=0x1018bab57530001 2024-12-07T06:45:09,450 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:45:09,450 DEBUG [RS:0;61c02eafbb40:37853 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,450 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,37853,1733553908997' 2024-12-07T06:45:09,450 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,450 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:45:09,450 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1482): Serving as 61c02eafbb40,38247,1733553909035, RpcServer on 61c02eafbb40/172.17.0.2:38247, sessionid=0x1018bab57530002 2024-12-07T06:45:09,450 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:45:09,450 DEBUG [RS:1;61c02eafbb40:38247 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c02eafbb40,38247,1733553909035 2024-12-07T06:45:09,451 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,38247,1733553909035' 2024-12-07T06:45:09,451 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:45:09,451 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:45:09,451 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:45:09,452 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:45:09,452 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:45:09,452 DEBUG [RS:0;61c02eafbb40:37853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,452 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,37853,1733553908997' 2024-12-07T06:45:09,452 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:45:09,452 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:45:09,452 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:45:09,452 DEBUG [RS:1;61c02eafbb40:38247 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c02eafbb40,38247,1733553909035 2024-12-07T06:45:09,452 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,38247,1733553909035' 2024-12-07T06:45:09,452 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:45:09,452 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:45:09,452 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:45:09,453 DEBUG [RS:0;61c02eafbb40:37853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:45:09,453 INFO [RS:0;61c02eafbb40:37853 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:45:09,453 INFO [RS:0;61c02eafbb40:37853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:45:09,453 DEBUG [RS:1;61c02eafbb40:38247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:45:09,453 INFO [RS:1;61c02eafbb40:38247 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:45:09,453 INFO [RS:1;61c02eafbb40:38247 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:45:09,476 WARN [61c02eafbb40:44987 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T06:45:09,554 INFO [RS:0;61c02eafbb40:37853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:45:09,554 INFO [RS:1;61c02eafbb40:38247 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:45:09,555 INFO [RS:0;61c02eafbb40:37853 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c02eafbb40%2C37853%2C1733553908997, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,37853,1733553908997, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs, maxLogs=55 2024-12-07T06:45:09,555 INFO [RS:1;61c02eafbb40:38247 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c02eafbb40%2C38247%2C1733553909035, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,38247,1733553909035, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs, maxLogs=55 2024-12-07T06:45:09,576 INFO [RS:1;61c02eafbb40:38247 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,38247,1733553909035/61c02eafbb40%2C38247%2C1733553909035.1733553909557 2024-12-07T06:45:09,577 DEBUG [RS:1;61c02eafbb40:38247 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:09,581 INFO [RS:0;61c02eafbb40:37853 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,37853,1733553908997/61c02eafbb40%2C37853%2C1733553908997.1733553909557 2024-12-07T06:45:09,581 DEBUG [RS:0;61c02eafbb40:37853 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:09,726 DEBUG [61c02eafbb40:44987 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T06:45:09,727 DEBUG [61c02eafbb40:44987 {}] balancer.BalancerClusterState(204): Hosts are {61c02eafbb40=0} racks are {/default-rack=0} 2024-12-07T06:45:09,728 DEBUG [61c02eafbb40:44987 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:45:09,728 DEBUG [61c02eafbb40:44987 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:45:09,728 DEBUG [61c02eafbb40:44987 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:45:09,728 DEBUG [61c02eafbb40:44987 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:45:09,728 INFO [61c02eafbb40:44987 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:45:09,728 INFO [61c02eafbb40:44987 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:45:09,728 DEBUG [61c02eafbb40:44987 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:45:09,728 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,729 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c02eafbb40,37853,1733553908997, state=OPENING 2024-12-07T06:45:09,733 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T06:45:09,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:09,735 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:45:09,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:09,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:09,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:09,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c02eafbb40,37853,1733553908997}] 2024-12-07T06:45:09,888 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:45:09,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47019, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:45:09,895 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T06:45:09,895 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:09,895 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T06:45:09,896 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c02eafbb40%2C37853%2C1733553908997.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,37853,1733553908997, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs, maxLogs=55 2024-12-07T06:45:09,925 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,37853,1733553908997/61c02eafbb40%2C37853%2C1733553908997.meta.1733553909898.meta 2024-12-07T06:45:09,926 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:09,926 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:09,927 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T06:45:09,927 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T06:45:09,927 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T06:45:09,927 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T06:45:09,927 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:09,927 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T06:45:09,927 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T06:45:09,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:45:09,931 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:45:09,931 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:45:09,933 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:45:09,933 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:45:09,934 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:45:09,934 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:45:09,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:45:09,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:09,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:45:09,936 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:45:09,937 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:09,937 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740 2024-12-07T06:45:09,938 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:45:09,938 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:45:09,938 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:45:09,939 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:45:09,940 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64929181, jitterRate=-0.03247980773448944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:45:09,940 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T06:45:09,941 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733553909928Writing region info on filesystem at 1733553909928Initializing all the Stores at 1733553909928Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553909928Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553909930 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553909930Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733553909930Cleaning up temporary data from old regions at 1733553909938 (+8 ms)Running coprocessor post-open hooks at 1733553909940 (+2 ms)Region opened successfully at 1733553909941 (+1 ms) 2024-12-07T06:45:09,942 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733553909888 2024-12-07T06:45:09,946 DEBUG [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T06:45:09,946 INFO [RS_OPEN_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T06:45:09,946 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,947 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c02eafbb40,37853,1733553908997, state=OPEN 2024-12-07T06:45:09,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:45:09,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:45:09,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:09,950 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c02eafbb40,37853,1733553908997 2024-12-07T06:45:09,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:09,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:45:09,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:45:09,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T06:45:09,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c02eafbb40,37853,1733553908997 in 215 msec 2024-12-07T06:45:09,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T06:45:09,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 629 msec 2024-12-07T06:45:09,956 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:45:09,956 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T06:45:09,969 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:45:09,969 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c02eafbb40,37853,1733553908997, seqNum=-1] 2024-12-07T06:45:09,970 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:45:09,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42983, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:45:09,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 708 msec 2024-12-07T06:45:09,981 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733553909981, completionTime=-1 2024-12-07T06:45:09,981 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T06:45:09,981 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T06:45:09,985 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T06:45:09,985 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733553969985 2024-12-07T06:45:09,985 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733554029985 2024-12-07T06:45:09,985 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-07T06:45:09,985 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,44987,1733553908946-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,986 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,44987,1733553908946-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,986 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,44987,1733553908946-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,986 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c02eafbb40:44987, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,986 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:09,989 DEBUG [master/61c02eafbb40:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T06:45:09,990 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:10,000 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.921sec 2024-12-07T06:45:10,000 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T06:45:10,000 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T06:45:10,001 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T06:45:10,001 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T06:45:10,001 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T06:45:10,001 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,44987,1733553908946-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:45:10,001 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,44987,1733553908946-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T06:45:10,002 DEBUG [master/61c02eafbb40:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T06:45:10,002 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T06:45:10,002 INFO [master/61c02eafbb40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,44987,1733553908946-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:10,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:45:10,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c02eafbb40,44987,-1 for getting cluster id 2024-12-07T06:45:10,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T06:45:10,062 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7c08351d-3ba5-4c0a-a882-7f77fcc5b4e8' 2024-12-07T06:45:10,063 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T06:45:10,063 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7c08351d-3ba5-4c0a-a882-7f77fcc5b4e8" 2024-12-07T06:45:10,064 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c8fc4f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:45:10,064 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c02eafbb40,44987,-1] 2024-12-07T06:45:10,064 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T06:45:10,065 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:10,067 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T06:45:10,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:45:10,069 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:45:10,071 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c02eafbb40,37853,1733553908997, seqNum=-1] 2024-12-07T06:45:10,072 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:45:10,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:45:10,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c02eafbb40,44987,1733553908946 2024-12-07T06:45:10,080 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T06:45:10,081 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c02eafbb40,44987,1733553908946 2024-12-07T06:45:10,081 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28ddeb83 2024-12-07T06:45:10,082 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T06:45:10,084 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T06:45:10,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44987 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733553910079', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:45:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44987 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 2024-12-07T06:45:10,088 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T06:45:10,088 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:10,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44987 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733553910079" procId is: 4 2024-12-07T06:45:10,090 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T06:45:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:10,114 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 32a99e1c406761aadecb7471b39bf947, NAME => 'testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733553910079', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:10,123 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:10,123 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(1722): Closing 32a99e1c406761aadecb7471b39bf947, disabling compactions & flushes 2024-12-07T06:45:10,124 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:10,124 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:10,124 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. after waiting 0 ms 2024-12-07T06:45:10,124 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:10,124 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:10,124 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733553910079-pool-0 {}] regionserver.HRegion(1676): Region close journal for 32a99e1c406761aadecb7471b39bf947: Waiting for close lock at 1733553910123Disabling compacts and flushes for region at 1733553910123Disabling writes for close at 1733553910124 (+1 ms)Writing region close event to WAL at 1733553910124Closed at 1733553910124 2024-12-07T06:45:10,126 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T06:45:10,126 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733553910126"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733553910126"}]},"ts":"1733553910126"} 2024-12-07T06:45:10,129 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T06:45:10,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T06:45:10,131 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733553910079","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733553910130"}]},"ts":"1733553910130"} 2024-12-07T06:45:10,134 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733553910079, state=ENABLING in hbase:meta 2024-12-07T06:45:10,135 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {61c02eafbb40=0} racks are {/default-rack=0} 2024-12-07T06:45:10,135 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:45:10,135 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:45:10,135 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:45:10,135 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:45:10,135 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:45:10,135 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:45:10,136 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:45:10,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN}] 2024-12-07T06:45:10,137 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN 2024-12-07T06:45:10,138 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN; state=OFFLINE, location=61c02eafbb40,38247,1733553909035; forceNewPlan=false, retain=false 2024-12-07T06:45:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:10,289 INFO [61c02eafbb40:44987 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T06:45:10,290 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=32a99e1c406761aadecb7471b39bf947, regionState=OPENING, regionLocation=61c02eafbb40,38247,1733553909035 2024-12-07T06:45:10,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN because future has completed 2024-12-07T06:45:10,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32a99e1c406761aadecb7471b39bf947, server=61c02eafbb40,38247,1733553909035}] 2024-12-07T06:45:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:10,446 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:45:10,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:45:10,451 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:10,451 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 32a99e1c406761aadecb7471b39bf947, NAME => 'testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:10,452 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733553910079 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,452 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:10,452 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,452 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,453 INFO [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,458 INFO [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32a99e1c406761aadecb7471b39bf947 columnFamilyName family 2024-12-07T06:45:10,458 DEBUG [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:10,458 INFO [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] regionserver.HStore(327): Store=32a99e1c406761aadecb7471b39bf947/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:10,459 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,459 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,459 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,460 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,460 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,460 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,470 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:45:10,471 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 32a99e1c406761aadecb7471b39bf947; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71301241, jitterRate=0.062471285462379456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:45:10,471 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:10,472 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 32a99e1c406761aadecb7471b39bf947: Running coprocessor pre-open hook at 1733553910452Writing region info on filesystem at 1733553910452Initializing all the Stores at 1733553910453 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553910453Cleaning up temporary data from old regions at 1733553910460 (+7 ms)Running coprocessor post-open hooks at 1733553910471 (+11 ms)Region opened successfully at 1733553910472 (+1 ms) 2024-12-07T06:45:10,473 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947., pid=6, masterSystemTime=1733553910446 2024-12-07T06:45:10,476 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:10,476 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:10,476 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=32a99e1c406761aadecb7471b39bf947, regionState=OPEN, openSeqNum=2, regionLocation=61c02eafbb40,38247,1733553909035 2024-12-07T06:45:10,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32a99e1c406761aadecb7471b39bf947, server=61c02eafbb40,38247,1733553909035 because future has completed 2024-12-07T06:45:10,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T06:45:10,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 32a99e1c406761aadecb7471b39bf947, server=61c02eafbb40,38247,1733553909035 in 187 msec 2024-12-07T06:45:10,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T06:45:10,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN in 346 msec 2024-12-07T06:45:10,484 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T06:45:10,485 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733553910079","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733553910484"}]},"ts":"1733553910484"} 2024-12-07T06:45:10,487 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733553910079, state=ENABLED in hbase:meta 2024-12-07T06:45:10,488 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T06:45:10,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733553910079 in 403 msec 2024-12-07T06:45:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:45:10,729 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733553910079 completed 2024-12-07T06:45:10,729 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733553910079 get assigned. Timeout = 60000ms 2024-12-07T06:45:10,729 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:45:10,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733553910079 assigned to meta. Checking AM states. 2024-12-07T06:45:10,733 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:45:10,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733553910079 assigned. 2024-12-07T06:45:10,737 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733553910079', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947., hostname=61c02eafbb40,38247,1733553909035, seqNum=2] 2024-12-07T06:45:10,737 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:45:10,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:45:10,744 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 32a99e1c406761aadecb7471b39bf947 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T06:45:10,754 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ee396bc339f84b3694585433745eed5a 2024-12-07T06:45:10,768 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/.tmp/family/ee396bc339f84b3694585433745eed5a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T06:45:10,768 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/.tmp/family/ee396bc339f84b3694585433745eed5a 2024-12-07T06:45:10,771 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/.tmp/family/ee396bc339f84b3694585433745eed5a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a 2024-12-07T06:45:10,773 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a, delay=907 ms 2024-12-07T06:45:10,774 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T06:45:10,775 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 32a99e1c406761aadecb7471b39bf947 in 32ms, sequenceid=6, compaction requested=false 2024-12-07T06:45:10,775 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733553910079' 2024-12-07T06:45:10,776 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 32a99e1c406761aadecb7471b39bf947: 2024-12-07T06:45:11,179 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T06:45:11,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T06:45:11,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-07T06:45:11,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T06:45:11,226 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-07T06:45:11,227 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-07T06:45:11,227 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-07T06:45:11,227 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733553910079 2024-12-07T06:45:11,227 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733553910079 Metrics about Tables on a single HBase RegionServer 2024-12-07T06:45:11,681 DEBUG [hfile-prefetch-1733553908379 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ee396bc339f84b3694585433745eed5a_0. 2024-12-07T06:45:11,682 DEBUG [hfile-prefetch-1733553908379 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ee396bc339f84b3694585433745eed5a_117. 2024-12-07T06:45:11,682 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a, with total blocks 2, and data blocks 1 2024-12-07T06:45:11,682 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T06:45:11,682 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T06:45:11,682 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a. Will sleep 100ms and try the verification again. 2024-12-07T06:45:11,709 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T06:45:11,709 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T06:45:11,709 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:11,709 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:45:11,713 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:45:11,713 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733553911709 2024-12-07T06:45:11,713 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T06:45:11,776 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;61c02eafbb40:38247,5,FailOnTimeoutGroup] 2024-12-07T06:45:11,776 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c02eafbb40,38247,1733553909035' ***** 2024-12-07T06:45:11,776 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-07T06:45:11,777 INFO [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:45:11,777 INFO [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence 2024-12-07T06:45:11,777 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:45:11,777 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:45:11,777 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:45:11,777 INFO [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:45:11,777 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:45:11,782 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a, with total blocks 2, and data blocks 1 2024-12-07T06:45:11,782 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-07T06:45:11,782 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2293): found block for file ee396bc339f84b3694585433745eed5a in the backing map. Acquiring read lock for offset 0 2024-12-07T06:45:11,782 DEBUG [hfile-prefetch-1733553908379 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a has now been fully cached. 2024-12-07T06:45:11,783 DEBUG [hfile-prefetch-1733553908379 {}] hfile.PrefetchExecutor(121): Prefetch completed for ee396bc339f84b3694585433745eed5a 2024-12-07T06:45:11,787 DEBUG [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1487): Thread RS:1;61c02eafbb40:38247 started persisting bucket cache to file 2024-12-07T06:45:11,787 DEBUG [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:11,787 DEBUG [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-07T06:45:11,795 DEBUG [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-07T06:45:11,795 DEBUG [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733553911787 2024-12-07T06:45:11,795 DEBUG [RS:1;61c02eafbb40:38247 {}] bucket.BucketCache(1509): Thread RS:1;61c02eafbb40:38247 finished persisting bucket cache to file, renaming 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:45:11,796 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(3091): Received CLOSE for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(959): stopping server 61c02eafbb40,38247,1733553909035 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;61c02eafbb40:38247. 2024-12-07T06:45:11,796 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:45:11,796 DEBUG [RS:1;61c02eafbb40:38247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:11,796 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32a99e1c406761aadecb7471b39bf947, disabling compactions & flushes 2024-12-07T06:45:11,796 DEBUG [RS:1;61c02eafbb40:38247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:11,796 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:11,796 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:11,796 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. after waiting 0 ms 2024-12-07T06:45:11,796 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:11,796 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-07T06:45:11,797 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1325): Online Regions={32a99e1c406761aadecb7471b39bf947=testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.} 2024-12-07T06:45:11,797 DEBUG [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1351): Waiting on 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:11,801 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T06:45:11,802 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:11,802 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32a99e1c406761aadecb7471b39bf947: Waiting for close lock at 1733553911796Running coprocessor pre-close hooks at 1733553911796Disabling compacts and flushes for region at 1733553911796Disabling writes for close at 1733553911796Writing region close event to WAL at 1733553911797 (+1 ms)Running coprocessor post-close hooks at 1733553911802 (+5 ms)Closed at 1733553911802 2024-12-07T06:45:11,802 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:11,821 INFO [regionserver/61c02eafbb40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:11,997 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(976): stopping server 61c02eafbb40,38247,1733553909035; all regions closed. 2024-12-07T06:45:11,999 DEBUG [RS:1;61c02eafbb40:38247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c02eafbb40%2C38247%2C1733553909035:(num 1733553909557) 2024-12-07T06:45:11,999 DEBUG [RS:1;61c02eafbb40:38247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] hbase.ChoreService(370): Chore service for: regionserver/61c02eafbb40:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:45:11,999 INFO [RS:1;61c02eafbb40:38247 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:45:11,999 INFO [regionserver/61c02eafbb40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:45:12,000 INFO [RS:1;61c02eafbb40:38247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38247 2024-12-07T06:45:12,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c02eafbb40,38247,1733553909035 2024-12-07T06:45:12,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:12,004 INFO [RS:1;61c02eafbb40:38247 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:45:12,006 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c02eafbb40,38247,1733553909035] 2024-12-07T06:45:12,008 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c02eafbb40,38247,1733553909035 already deleted, retry=false 2024-12-07T06:45:12,009 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 61c02eafbb40,38247,1733553909035 on 61c02eafbb40,44987,1733553908946 2024-12-07T06:45:12,013 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 61c02eafbb40,38247,1733553909035, splitWal=true, meta=false 2024-12-07T06:45:12,015 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 61c02eafbb40,38247,1733553909035 (carryingMeta=false) 61c02eafbb40,38247,1733553909035/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5eb9a108[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-07T06:45:12,015 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 61c02eafbb40,38247,1733553909035, splitWal=true, meta=false 2024-12-07T06:45:12,017 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 61c02eafbb40,38247,1733553909035 had 1 regions 2024-12-07T06:45:12,017 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 61c02eafbb40,38247,1733553909035, splitWal=true, meta=false, isMeta: false 2024-12-07T06:45:12,018 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,38247,1733553909035-splitting 2024-12-07T06:45:12,018 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,38247,1733553909035-splitting dir is empty, no logs to split. 2024-12-07T06:45:12,018 INFO [PEWorker-2 {}] master.SplitWALManager(105): 61c02eafbb40,38247,1733553909035 WAL count=0, meta=false 2024-12-07T06:45:12,020 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,38247,1733553909035-splitting dir is empty, no logs to split. 2024-12-07T06:45:12,020 INFO [PEWorker-2 {}] master.SplitWALManager(105): 61c02eafbb40,38247,1733553909035 WAL count=0, meta=false 2024-12-07T06:45:12,020 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 61c02eafbb40,38247,1733553909035 WAL splitting is done? wals=0, meta=false 2024-12-07T06:45:12,020 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 61c02eafbb40,38247,1733553909035 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,38247,1733553909035-splitting does not exist 2024-12-07T06:45:12,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN}] 2024-12-07T06:45:12,023 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN 2024-12-07T06:45:12,023 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T06:45:12,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:12,106 INFO [RS:1;61c02eafbb40:38247 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:45:12,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38247-0x1018bab57530002, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:12,106 INFO [RS:1;61c02eafbb40:38247 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c02eafbb40,38247,1733553909035; zookeeper connection closed. 2024-12-07T06:45:12,106 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3400c4c1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3400c4c1 2024-12-07T06:45:12,174 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=32a99e1c406761aadecb7471b39bf947, regionState=OPENING, regionLocation=61c02eafbb40,37853,1733553908997 2024-12-07T06:45:12,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN because future has completed 2024-12-07T06:45:12,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32a99e1c406761aadecb7471b39bf947, server=61c02eafbb40,37853,1733553908997}] 2024-12-07T06:45:12,293 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c02eafbb40:0 server-side Connection retries=45 2024-12-07T06:45:12,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:12,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:12,293 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:45:12,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:45:12,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:45:12,293 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:45:12,293 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:45:12,294 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35981 2024-12-07T06:45:12,295 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35981 connecting to ZooKeeper ensemble=127.0.0.1:51898 2024-12-07T06:45:12,295 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:12,295 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:12,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359810x0, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:45:12,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35981-0x1018bab57530003 connected 2024-12-07T06:45:12,299 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-07T06:45:12,299 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-07T06:45:12,300 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:45:12,300 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.cache 2024-12-07T06:45:12,300 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:45:12,300 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:45:12,300 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:45:12,301 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:45:12,301 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:45:12,301 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:45:12,302 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T06:45:12,302 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-07T06:45:12,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:45:12,303 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-12-07T06:45:12,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35981 2024-12-07T06:45:12,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35981 2024-12-07T06:45:12,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35981 2024-12-07T06:45:12,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35981 2024-12-07T06:45:12,307 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-07T06:45:12,307 DEBUG [Thread-200 {}] bucket.BucketCache(1584): Cached File Entry:> 2024-12-07T06:45:12,307 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:45:12,307 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-12-07T06:45:12,308 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence is complete. 2024-12-07T06:45:12,308 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35981 2024-12-07T06:45:12,310 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(746): ClusterId : 7c08351d-3ba5-4c0a-a882-7f77fcc5b4e8 2024-12-07T06:45:12,310 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:45:12,312 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:45:12,312 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:45:12,314 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:45:12,315 DEBUG [RS:2;61c02eafbb40:35981 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a7eac7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c02eafbb40/172.17.0.2:0 2024-12-07T06:45:12,327 DEBUG [RS:2;61c02eafbb40:35981 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;61c02eafbb40:35981 2024-12-07T06:45:12,327 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:45:12,327 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:45:12,327 DEBUG [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:45:12,328 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c02eafbb40,44987,1733553908946 with port=35981, startcode=1733553912292 2024-12-07T06:45:12,328 DEBUG [RS:2;61c02eafbb40:35981 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:45:12,330 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47503, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:45:12,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44987 {}] master.ServerManager(517): Registering regionserver=61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,333 DEBUG [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c 2024-12-07T06:45:12,333 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,333 DEBUG [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:45:12,333 DEBUG [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:45:12,333 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 32a99e1c406761aadecb7471b39bf947, NAME => 'testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:45:12,333 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733553910079 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,334 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:45:12,334 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,334 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,335 INFO [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,335 INFO [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32a99e1c406761aadecb7471b39bf947 columnFamilyName family 2024-12-07T06:45:12,336 DEBUG [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:45:12,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:12,337 DEBUG [RS:2;61c02eafbb40:35981 {}] zookeeper.ZKUtil(111): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,337 WARN [RS:2;61c02eafbb40:35981 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:45:12,337 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c02eafbb40,35981,1733553912292] 2024-12-07T06:45:12,337 INFO [RS:2;61c02eafbb40:35981 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:45:12,337 DEBUG [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,338 DEBUG [StoreFileOpener-32a99e1c406761aadecb7471b39bf947-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a, delay=914 ms 2024-12-07T06:45:12,339 DEBUG [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a 2024-12-07T06:45:12,339 INFO [StoreOpener-32a99e1c406761aadecb7471b39bf947-1 {}] regionserver.HStore(327): Store=32a99e1c406761aadecb7471b39bf947/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:45:12,339 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,340 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,340 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,341 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,341 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,342 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,342 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 32a99e1c406761aadecb7471b39bf947; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65164932, jitterRate=-0.028966844081878662}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:45:12,342 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,343 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:45:12,344 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 32a99e1c406761aadecb7471b39bf947: Running coprocessor pre-open hook at 1733553912334Writing region info on filesystem at 1733553912334Initializing all the Stores at 1733553912334Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733553912334Cleaning up temporary data from old regions at 1733553912341 (+7 ms)Running coprocessor post-open hooks at 1733553912343 (+2 ms)Region opened successfully at 1733553912344 (+1 ms) 2024-12-07T06:45:12,344 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947., pid=9, masterSystemTime=1733553912330 2024-12-07T06:45:12,346 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:45:12,347 INFO [RS:2;61c02eafbb40:35981 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:45:12,347 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,347 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:45:12,347 DEBUG [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,347 INFO [RS_OPEN_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,347 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:45:12,347 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=32a99e1c406761aadecb7471b39bf947, regionState=OPEN, openSeqNum=10, regionLocation=61c02eafbb40,37853,1733553908997 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c02eafbb40:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c02eafbb40:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:12,348 DEBUG [RS:2;61c02eafbb40:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c02eafbb40:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:45:12,349 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,349 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,349 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,349 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,349 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,350 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35981,1733553912292-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:45:12,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32a99e1c406761aadecb7471b39bf947, server=61c02eafbb40,37853,1733553908997 because future has completed 2024-12-07T06:45:12,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-07T06:45:12,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 32a99e1c406761aadecb7471b39bf947, server=61c02eafbb40,37853,1733553908997 in 173 msec 2024-12-07T06:45:12,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T06:45:12,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733553910079, region=32a99e1c406761aadecb7471b39bf947, ASSIGN in 331 msec 2024-12-07T06:45:12,355 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 61c02eafbb40,38247,1733553909035 after splitting done 2024-12-07T06:45:12,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 61c02eafbb40,38247,1733553909035, splitWal=true, meta=false in 345 msec 2024-12-07T06:45:12,371 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:45:12,371 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c02eafbb40,35981,1733553912292-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,371 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,371 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.Replication(171): 61c02eafbb40,35981,1733553912292 started 2024-12-07T06:45:12,385 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:45:12,386 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(1482): Serving as 61c02eafbb40,35981,1733553912292, RpcServer on 61c02eafbb40/172.17.0.2:35981, sessionid=0x1018bab57530003 2024-12-07T06:45:12,386 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:45:12,386 DEBUG [RS:2;61c02eafbb40:35981 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,386 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,35981,1733553912292' 2024-12-07T06:45:12,386 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:45:12,387 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:45:12,387 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:45:12,387 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:45:12,387 DEBUG [RS:2;61c02eafbb40:35981 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,387 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c02eafbb40,35981,1733553912292' 2024-12-07T06:45:12,387 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:45:12,388 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:45:12,388 DEBUG [RS:2;61c02eafbb40:35981 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:45:12,388 INFO [RS:2;61c02eafbb40:35981 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:45:12,388 INFO [RS:2;61c02eafbb40:35981 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:45:12,489 INFO [RS:2;61c02eafbb40:35981 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:45:12,489 INFO [RS:2;61c02eafbb40:35981 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c02eafbb40%2C35981%2C1733553912292, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,35981,1733553912292, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs, maxLogs=55 2024-12-07T06:45:12,507 INFO [RS:2;61c02eafbb40:35981 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/WALs/61c02eafbb40,35981,1733553912292/61c02eafbb40%2C35981%2C1733553912292.1733553912490 2024-12-07T06:45:12,508 DEBUG [RS:2;61c02eafbb40:35981 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:45:12,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T06:45:12,886 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:45:12,886 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:12,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:12,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:12,886 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T06:45:12,886 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T06:45:12,887 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1972179983, stopped=false 2024-12-07T06:45:12,887 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c02eafbb40,44987,1733553908946 2024-12-07T06:45:12,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:12,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:12,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:45:12,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:12,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:12,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:12,889 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:45:12,889 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:45:12,889 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:12,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:12,889 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c02eafbb40,37853,1733553908997' ***** 2024-12-07T06:45:12,889 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:45:12,889 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c02eafbb40,35981,1733553912292' ***** 2024-12-07T06:45:12,889 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:45:12,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:12,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:12,890 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:45:12,890 INFO [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:45:12,890 INFO [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:45:12,890 INFO [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence 2024-12-07T06:45:12,890 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:45:12,890 INFO [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eda91a9b-13b2-d3df-d68f-922ad828799c/bucket.persistence 2024-12-07T06:45:12,890 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:45:12,890 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:45:12,890 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:45:12,890 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:45:12,890 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:45:12,891 INFO [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:45:12,891 INFO [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:45:12,891 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:45:12,891 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:45:12,901 DEBUG [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1487): Thread RS:2;61c02eafbb40:35981 started persisting bucket cache to file 2024-12-07T06:45:12,901 DEBUG [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1487): Thread RS:0;61c02eafbb40:37853 started persisting bucket cache to file 2024-12-07T06:45:12,901 DEBUG [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:12,901 DEBUG [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-07T06:45:12,901 DEBUG [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:45:12,901 DEBUG [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:45:12,905 DEBUG [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:45:12,905 DEBUG [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733553912901 2024-12-07T06:45:12,905 DEBUG [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-07T06:45:12,905 DEBUG [RS:0;61c02eafbb40:37853 {}] bucket.BucketCache(1509): Thread RS:0;61c02eafbb40:37853 finished persisting bucket cache to file, renaming 2024-12-07T06:45:12,905 DEBUG [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733553912901 2024-12-07T06:45:12,905 DEBUG [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1509): Thread RS:2;61c02eafbb40:35981 finished persisting bucket cache to file, renaming 2024-12-07T06:45:12,905 WARN [RS:2;61c02eafbb40:35981 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-07T06:45:12,905 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:45:12,905 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:45:12,905 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:45:12,905 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:45:12,905 INFO [RS:2;61c02eafbb40:35981 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:45:12,905 INFO [RS:0;61c02eafbb40:37853 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:45:12,905 INFO [RS:2;61c02eafbb40:35981 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:45:12,905 INFO [RS:0;61c02eafbb40:37853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:45:12,905 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(959): stopping server 61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,905 INFO [RS:2;61c02eafbb40:35981 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:45:12,905 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(3091): Received CLOSE for 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,905 INFO [RS:2;61c02eafbb40:35981 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;61c02eafbb40:35981. 2024-12-07T06:45:12,905 DEBUG [RS:2;61c02eafbb40:35981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:12,905 DEBUG [RS:2;61c02eafbb40:35981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(959): stopping server 61c02eafbb40,37853,1733553908997 2024-12-07T06:45:12,906 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(976): stopping server 61c02eafbb40,35981,1733553912292; all regions closed. 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c02eafbb40:37853. 2024-12-07T06:45:12,906 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:45:12,906 DEBUG [RS:0;61c02eafbb40:37853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:45:12,906 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32a99e1c406761aadecb7471b39bf947, disabling compactions & flushes 2024-12-07T06:45:12,906 DEBUG [RS:0;61c02eafbb40:37853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:12,906 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,906 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,906 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. after waiting 0 ms 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:45:12,906 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T06:45:12,906 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T06:45:12,906 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 32a99e1c406761aadecb7471b39bf947=testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947.} 2024-12-07T06:45:12,906 DEBUG [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 32a99e1c406761aadecb7471b39bf947 2024-12-07T06:45:12,906 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:45:12,907 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:45:12,907 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:45:12,907 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:45:12,907 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:45:12,907 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:45:12,907 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-07T06:45:12,907 DEBUG [StoreFileCloser-32a99e1c406761aadecb7471b39bf947-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/family/ee396bc339f84b3694585433745eed5a 2024-12-07T06:45:12,908 DEBUG [RS:2;61c02eafbb40:35981 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs 2024-12-07T06:45:12,908 INFO [RS:2;61c02eafbb40:35981 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c02eafbb40%2C35981%2C1733553912292:(num 1733553912490) 2024-12-07T06:45:12,908 DEBUG [RS:2;61c02eafbb40:35981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:12,908 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:12,909 INFO [RS:2;61c02eafbb40:35981 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:45:12,909 INFO [RS:2;61c02eafbb40:35981 {}] hbase.ChoreService(370): Chore service for: regionserver/61c02eafbb40:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T06:45:12,909 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:45:12,909 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:45:12,909 INFO [regionserver/61c02eafbb40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:45:12,909 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:45:12,909 INFO [RS:2;61c02eafbb40:35981 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:45:12,909 INFO [RS:2;61c02eafbb40:35981 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35981 2024-12-07T06:45:12,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c02eafbb40,35981,1733553912292 2024-12-07T06:45:12,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:12,912 INFO [RS:2;61c02eafbb40:35981 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:45:12,913 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c02eafbb40,35981,1733553912292] 2024-12-07T06:45:12,914 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/default/testBlockEvictionOnGracefulStop1733553910079/32a99e1c406761aadecb7471b39bf947/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T06:45:12,915 INFO [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,915 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32a99e1c406761aadecb7471b39bf947: Waiting for close lock at 1733553912906Running coprocessor pre-close hooks at 1733553912906Disabling compacts and flushes for region at 1733553912906Disabling writes for close at 1733553912906Writing region close event to WAL at 1733553912909 (+3 ms)Running coprocessor post-close hooks at 1733553912915 (+6 ms)Closed at 1733553912915 2024-12-07T06:45:12,915 DEBUG [RS_CLOSE_REGION-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947. 2024-12-07T06:45:12,916 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c02eafbb40,35981,1733553912292 already deleted, retry=false 2024-12-07T06:45:12,916 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c02eafbb40,35981,1733553912292 expired; onlineServers=1 2024-12-07T06:45:12,918 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4fb16ff06a854a0f927f4afd1a94b6f6 2024-12-07T06:45:12,922 INFO [regionserver/61c02eafbb40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:12,933 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/info/4fb16ff06a854a0f927f4afd1a94b6f6 is 209, key is testBlockEvictionOnGracefulStop1733553910079,,1733553910085.32a99e1c406761aadecb7471b39bf947./info:regioninfo/1733553912347/Put/seqid=0 2024-12-07T06:45:12,934 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/info/4fb16ff06a854a0f927f4afd1a94b6f6 2024-12-07T06:45:12,943 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b6e3229eb87742139d267a90c0bed60c 2024-12-07T06:45:12,950 INFO [regionserver/61c02eafbb40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:12,958 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/ns/b6e3229eb87742139d267a90c0bed60c is 43, key is default/ns:d/1733553909974/Put/seqid=0 2024-12-07T06:45:12,959 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/ns/b6e3229eb87742139d267a90c0bed60c 2024-12-07T06:45:12,969 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46ca456de33043e4b1106dc1f5dc597c 2024-12-07T06:45:12,983 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/table/46ca456de33043e4b1106dc1f5dc597c is 80, key is testBlockEvictionOnGracefulStop1733553910079/table:state/1733553910484/Put/seqid=0 2024-12-07T06:45:12,984 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/table/46ca456de33043e4b1106dc1f5dc597c 2024-12-07T06:45:12,986 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/info/4fb16ff06a854a0f927f4afd1a94b6f6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/4fb16ff06a854a0f927f4afd1a94b6f6 2024-12-07T06:45:12,987 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/4fb16ff06a854a0f927f4afd1a94b6f6, delay=942 ms 2024-12-07T06:45:12,988 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/4fb16ff06a854a0f927f4afd1a94b6f6, entries=14, sequenceid=13, filesize=7.5 K 2024-12-07T06:45:12,989 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/ns/b6e3229eb87742139d267a90c0bed60c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/b6e3229eb87742139d267a90c0bed60c 2024-12-07T06:45:12,990 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/b6e3229eb87742139d267a90c0bed60c, delay=934 ms 2024-12-07T06:45:12,991 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/b6e3229eb87742139d267a90c0bed60c, entries=2, sequenceid=13, filesize=5.0 K 2024-12-07T06:45:12,991 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/.tmp/table/46ca456de33043e4b1106dc1f5dc597c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46ca456de33043e4b1106dc1f5dc597c 2024-12-07T06:45:12,992 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46ca456de33043e4b1106dc1f5dc597c, delay=952 ms 2024-12-07T06:45:12,993 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46ca456de33043e4b1106dc1f5dc597c, entries=2, sequenceid=13, filesize=5.3 K 2024-12-07T06:45:12,994 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false 2024-12-07T06:45:12,995 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/info/4fb16ff06a854a0f927f4afd1a94b6f6 2024-12-07T06:45:12,995 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/ns/b6e3229eb87742139d267a90c0bed60c 2024-12-07T06:45:12,996 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/table/46ca456de33043e4b1106dc1f5dc597c 2024-12-07T06:45:13,000 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-07T06:45:13,000 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T06:45:13,000 INFO [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:45:13,001 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733553912906Running coprocessor pre-close hooks at 1733553912907 (+1 ms)Disabling compacts and flushes for region at 1733553912907Disabling writes for close at 1733553912907Obtaining lock to block concurrent updates at 1733553912907Preparing flush snapshotting stores in 1588230740 at 1733553912907Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733553912907Flushing stores of hbase:meta,,1.1588230740 at 1733553912908 (+1 ms)Flushing 1588230740/info: creating writer at 1733553912908Flushing 1588230740/info: appending metadata at 1733553912933 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733553912933Flushing 1588230740/ns: creating writer at 1733553912936 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733553912958 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733553912958Flushing 1588230740/table: creating writer at 1733553912962 (+4 ms)Flushing 1588230740/table: appending metadata at 1733553912982 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733553912982Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@812575d: reopening flushed file at 1733553912986 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f5a8f37: reopening flushed file at 1733553912989 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3da2177b: reopening flushed file at 1733553912991 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false at 1733553912994 (+3 ms)Writing region close event to WAL at 1733553912996 (+2 ms)Running coprocessor post-close hooks at 1733553913000 (+4 ms)Closed at 1733553913000 2024-12-07T06:45:13,001 DEBUG [RS_CLOSE_META-regionserver/61c02eafbb40:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T06:45:13,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:13,013 INFO [RS:2;61c02eafbb40:35981 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:45:13,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1018bab57530003, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:13,014 INFO [RS:2;61c02eafbb40:35981 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c02eafbb40,35981,1733553912292; zookeeper connection closed. 2024-12-07T06:45:13,014 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@15490dff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@15490dff 2024-12-07T06:45:13,107 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(976): stopping server 61c02eafbb40,37853,1733553908997; all regions closed. 2024-12-07T06:45:13,109 DEBUG [RS:0;61c02eafbb40:37853 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs 2024-12-07T06:45:13,109 INFO [RS:0;61c02eafbb40:37853 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c02eafbb40%2C37853%2C1733553908997.meta:.meta(num 1733553909898) 2024-12-07T06:45:13,110 DEBUG [RS:0;61c02eafbb40:37853 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/oldWALs 2024-12-07T06:45:13,110 INFO [RS:0;61c02eafbb40:37853 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c02eafbb40%2C37853%2C1733553908997:(num 1733553909557) 2024-12-07T06:45:13,110 DEBUG [RS:0;61c02eafbb40:37853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:45:13,110 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:45:13,110 INFO [RS:0;61c02eafbb40:37853 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:45:13,111 INFO [RS:0;61c02eafbb40:37853 {}] hbase.ChoreService(370): Chore service for: regionserver/61c02eafbb40:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T06:45:13,111 INFO [RS:0;61c02eafbb40:37853 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:45:13,111 INFO [regionserver/61c02eafbb40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:45:13,111 INFO [RS:0;61c02eafbb40:37853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37853 2024-12-07T06:45:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c02eafbb40,37853,1733553908997 2024-12-07T06:45:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:45:13,113 INFO [RS:0;61c02eafbb40:37853 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:45:13,115 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c02eafbb40,37853,1733553908997] 2024-12-07T06:45:13,116 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c02eafbb40,37853,1733553908997 already deleted, retry=false 2024-12-07T06:45:13,116 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c02eafbb40,37853,1733553908997 expired; onlineServers=0 2024-12-07T06:45:13,117 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c02eafbb40,44987,1733553908946' ***** 2024-12-07T06:45:13,117 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T06:45:13,117 INFO [M:0;61c02eafbb40:44987 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:45:13,117 INFO [M:0;61c02eafbb40:44987 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:45:13,117 DEBUG [M:0;61c02eafbb40:44987 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T06:45:13,117 DEBUG [M:0;61c02eafbb40:44987 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T06:45:13,117 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T06:45:13,117 DEBUG [master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.small.0-1733553909281 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.small.0-1733553909281,5,FailOnTimeoutGroup] 2024-12-07T06:45:13,117 DEBUG [master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.large.0-1733553909281 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c02eafbb40:0:becomeActiveMaster-HFileCleaner.large.0-1733553909281,5,FailOnTimeoutGroup] 2024-12-07T06:45:13,117 INFO [M:0;61c02eafbb40:44987 {}] hbase.ChoreService(370): Chore service for: master/61c02eafbb40:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T06:45:13,117 INFO [M:0;61c02eafbb40:44987 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:45:13,118 DEBUG [M:0;61c02eafbb40:44987 {}] master.HMaster(1795): Stopping service threads 2024-12-07T06:45:13,118 INFO [M:0;61c02eafbb40:44987 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T06:45:13,118 INFO [M:0;61c02eafbb40:44987 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:45:13,118 INFO [M:0;61c02eafbb40:44987 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T06:45:13,118 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T06:45:13,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T06:45:13,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:45:13,120 DEBUG [M:0;61c02eafbb40:44987 {}] zookeeper.ZKUtil(347): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T06:45:13,120 WARN [M:0;61c02eafbb40:44987 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T06:45:13,121 INFO [M:0;61c02eafbb40:44987 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/.lastflushedseqids 2024-12-07T06:45:13,127 INFO [M:0;61c02eafbb40:44987 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T06:45:13,128 INFO [M:0;61c02eafbb40:44987 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T06:45:13,128 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:45:13,128 INFO [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:13,128 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:13,128 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:45:13,128 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:13,128 INFO [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-12-07T06:45:13,139 DEBUG [M:0;61c02eafbb40:44987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a82cc657b97345de96c13cfd80f4d5df 2024-12-07T06:45:13,152 DEBUG [M:0;61c02eafbb40:44987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a82cc657b97345de96c13cfd80f4d5df is 82, key is hbase:meta,,1/info:regioninfo/1733553909946/Put/seqid=0 2024-12-07T06:45:13,153 INFO [M:0;61c02eafbb40:44987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a82cc657b97345de96c13cfd80f4d5df 2024-12-07T06:45:13,161 DEBUG [M:0;61c02eafbb40:44987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0c98e31fe91b4f4592bf51f4146f4cc0 2024-12-07T06:45:13,174 DEBUG [M:0;61c02eafbb40:44987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c98e31fe91b4f4592bf51f4146f4cc0 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733553910488/Put/seqid=0 2024-12-07T06:45:13,175 INFO [M:0;61c02eafbb40:44987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c98e31fe91b4f4592bf51f4146f4cc0 2024-12-07T06:45:13,183 DEBUG [M:0;61c02eafbb40:44987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 125ee54060154ea498f12581a2eacf8a 2024-12-07T06:45:13,197 DEBUG [M:0;61c02eafbb40:44987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/125ee54060154ea498f12581a2eacf8a is 69, key is 61c02eafbb40,35981,1733553912292/rs:state/1733553912331/Put/seqid=0 2024-12-07T06:45:13,197 INFO [M:0;61c02eafbb40:44987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/125ee54060154ea498f12581a2eacf8a 2024-12-07T06:45:13,199 INFO [M:0;61c02eafbb40:44987 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 125ee54060154ea498f12581a2eacf8a 2024-12-07T06:45:13,199 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a82cc657b97345de96c13cfd80f4d5df as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a82cc657b97345de96c13cfd80f4d5df 2024-12-07T06:45:13,201 INFO [M:0;61c02eafbb40:44987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a82cc657b97345de96c13cfd80f4d5df, entries=8, sequenceid=83, filesize=5.5 K 2024-12-07T06:45:13,202 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c98e31fe91b4f4592bf51f4146f4cc0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c98e31fe91b4f4592bf51f4146f4cc0 2024-12-07T06:45:13,203 INFO [M:0;61c02eafbb40:44987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c98e31fe91b4f4592bf51f4146f4cc0, entries=9, sequenceid=83, filesize=6.4 K 2024-12-07T06:45:13,203 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/125ee54060154ea498f12581a2eacf8a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/125ee54060154ea498f12581a2eacf8a 2024-12-07T06:45:13,205 INFO [M:0;61c02eafbb40:44987 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 125ee54060154ea498f12581a2eacf8a 2024-12-07T06:45:13,205 INFO [M:0;61c02eafbb40:44987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1c0e1c8-eb17-7100-58ea-343250d67190/d5e84fac-b2bd-af4c-2f30-3ac7565f477c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/125ee54060154ea498f12581a2eacf8a, entries=3, sequenceid=83, filesize=5.3 K 2024-12-07T06:45:13,206 INFO [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false 2024-12-07T06:45:13,207 INFO [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:45:13,208 DEBUG [M:0;61c02eafbb40:44987 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733553913128Disabling compacts and flushes for region at 1733553913128Disabling writes for close at 1733553913128Obtaining lock to block concurrent updates at 1733553913128Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733553913128Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1733553913129 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733553913129Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733553913129Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733553913152 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733553913152Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733553913155 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733553913174 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733553913174Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733553913177 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733553913196 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733553913196Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ceeab1: reopening flushed file at 1733553913199 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f141efa: reopening flushed file at 1733553913201 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6211d97c: reopening flushed file at 1733553913203 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false at 1733553913206 (+3 ms)Writing region close event to WAL at 1733553913207 (+1 ms)Closed at 1733553913207 2024-12-07T06:45:13,209 INFO [M:0;61c02eafbb40:44987 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T06:45:13,209 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:45:13,209 INFO [M:0;61c02eafbb40:44987 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44987 2024-12-07T06:45:13,209 INFO [M:0;61c02eafbb40:44987 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:45:13,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:13,215 INFO [RS:0;61c02eafbb40:37853 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:45:13,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1018bab57530001, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:13,215 INFO [RS:0;61c02eafbb40:37853 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c02eafbb40,37853,1733553908997; zookeeper connection closed. 2024-12-07T06:45:13,216 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57968710 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57968710 2024-12-07T06:45:13,216 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-07T06:45:13,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:13,312 INFO [M:0;61c02eafbb40:44987 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:45:13,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44987-0x1018bab57530000, quorum=127.0.0.1:51898, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:45:13,314 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T06:45:13,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T06:45:13,316 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:45:13,322 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=4928 (was 5604)